Skip to main content

formualizer_eval/engine/
eval.rs

1use crate::SheetId;
2
3mod criteria;
4mod criteria_mask_cache;
5#[cfg(test)]
6mod criteria_mask_cache_tests;
7mod exact;
8mod family;
9mod kernels;
10mod lift;
11mod memo;
12use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
13#[cfg(test)]
14use crate::engine::Scheduler;
15use crate::engine::arena::AstNodeId;
16use crate::engine::eval_delta::{
17    DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
18};
19use crate::engine::graph::editor::change_log::MutationCapture;
20use crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan;
21use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
22use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
23use crate::engine::live_graph::analyze_live_graph;
24use crate::engine::lookup_index_cache::{
25    BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
26    LookupIndexKey, estimate_bytes,
27};
28use crate::engine::named_range::{NameScope, NamedDefinition};
29use crate::engine::range_view::RangeView;
30use crate::engine::row_visibility::RowVisibilityState;
31use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
32use crate::engine::target_preparation::{
33    StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
34};
35use crate::engine::used_extent::{
36    ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
37};
38use crate::engine::virtual_deps::VirtualDepBuilder;
39use family::{LayerUnit, layer_units, unit_members};
40
41#[path = "freshness.rs"]
42mod freshness;
43use crate::engine::template::region::Region;
44use crate::engine::{
45    ChangeLogger, CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
46    EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceRequestStats,
47    FormulaDirtyLeaseOutcome, FormulaIngestBatch, FormulaIngestRecord, FormulaIngestReport,
48    FormulaParseDiagnostic, FormulaParsePolicy, FormulaPlaneMode, ResourceLedger,
49    RowVisibilitySource, ScheduleUnit, VertexId, VertexKind, VisibilityMaskMode,
50};
51use crate::function::FnCaps;
52use crate::interpreter::Interpreter;
53
54#[cfg(test)]
55mod fn_snapshot_tests;
56use crate::reference::{CellRef, Coord, RangeRef};
57use crate::traits::FunctionProvider;
58use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
59use formualizer_common::{
60    CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
61};
62use formualizer_parse::parser::ReferenceType;
63use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
64use rayon::ThreadPoolBuilder;
65use rustc_hash::{FxHashMap, FxHashSet};
66use std::collections::{BTreeMap, BTreeSet, VecDeque};
67use std::sync::Arc;
68use std::sync::atomic::{AtomicBool, Ordering};
69use std::time::Instant;
70
71type StagedFormulaEntry = (u32, u32, String);
72type StagedSheetParts = (
73    Vec<StagedFormulaEntry>,
74    Option<crate::engine::DeferredFormulaPackage>,
75);
76
77/// Per-sheet staged-formula store (NOTE(#126) follow-up).
78///
79/// Ingest consumers (`build_graph_all`/`build_graph_for_sheets`) walk staged
80/// entries in INSERTION order, so the order-preserving `Vec` stays the
81/// canonical storage; a `(row, col) → index` map removes the linear dup-scan
82/// that made `stage_formula_text`/`get_staged_formula_text` O(staged-on-sheet)
83/// per call (O(n²) for an n-formula deferred load on one sheet — ~570 ms for
84/// 50k stages, release, before the index). `stage`/`get` are O(1);
85/// `remove` keeps the old O(n) `Vec::remove` (rare path, order preserved).
86#[derive(Default)]
87pub(crate) struct StagedSheet {
88    entries: Vec<StagedFormulaEntry>,
89    index: FxHashMap<(u32, u32), usize>,
90    deferred_package: Option<crate::engine::DeferredFormulaPackage>,
91}
92
93impl StagedSheet {
94    fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
95        package
96            .invalidated
97            .extend(package.families.iter().map(|family| family.source_id));
98        package.invalidated.extend(
99            package
100                .partitioned_families
101                .iter()
102                .map(|family| family.source_id),
103        );
104    }
105
106    fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
107        let Some(package) = self.deferred_package.as_mut() else {
108            return;
109        };
110        if package.suppressed.contains(&(row, col)) {
111            // Previously consumed coordinates are already excluded from residual
112            // authority. Edits/deletion must not invalidate untouched fragments.
113            return;
114        }
115        if package.source_geometry_complete
116            && package.families.is_empty()
117            && package.partitioned_families.is_empty()
118        {
119            if let Some((row0, col0)) = row.checked_sub(1).zip(col.checked_sub(1))
120                && package
121                    .source_coordinates
122                    .binary_search(&crate::engine::SourceCoord {
123                        row: row0,
124                        col: col0,
125                    })
126                    .is_ok()
127            {
128                package.suppressed.insert((row, col));
129            }
130            return;
131        }
132        let family = package
133            .replay
134            .lock()
135            .ok()
136            .and_then(|mut replay| replay.formula_at(row, col).ok())
137            .flatten()
138            .and_then(|record| record.partition_owner.or(record.family));
139        if let Some(family) = family {
140            package.invalidated.insert(family);
141        } else {
142            // A poisoned lock, replay failure, malformed spool, or missing
143            // lookup result must never let a possibly edited family commit.
144            // Fail closed at package scope before suppressing the edited cell.
145            Self::invalidate_all_deferred_families(package);
146        }
147        package.suppressed.insert((row, col));
148    }
149
150    fn reconcile_attached_deferred_package(&mut self) {
151        let Some(package) = self.deferred_package.as_mut() else {
152            return;
153        };
154        if self.entries.is_empty() {
155            return;
156        }
157
158        let replayed = (|| {
159            let mut disposition = crate::engine::FormulaReplayDisposition::default();
160            for partition in &package.partitioned_families {
161                disposition.register_partition(partition, false)?;
162            }
163            package
164                .replay
165                .lock()
166                .map_err(|_| "deferred formula spool lock poisoned".to_string())?
167                .replay_partitioned(&disposition, &package.partitioned_families)
168        })();
169
170        let Ok(records) = replayed else {
171            Self::invalidate_all_deferred_families(package);
172            package
173                .suppressed
174                .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
175            return;
176        };
177        let mut owners = FxHashMap::default();
178        for record in &records {
179            owners
180                .entry((record.row, record.col))
181                .or_insert(record.partition_owner.or(record.family));
182        }
183        for (row, col, _) in &self.entries {
184            if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
185                package.invalidated.insert(family);
186            } else {
187                Self::invalidate_all_deferred_families(package);
188            }
189            package.suppressed.insert((*row, *col));
190        }
191        package.reconciliation_replay = Some(records);
192    }
193
194    fn stage(&mut self, row: u32, col: u32, text: String) {
195        self.invalidate_deferred_at(row, col);
196        match self.index.entry((row, col)) {
197            std::collections::hash_map::Entry::Occupied(slot) => {
198                self.entries[*slot.get()].2 = text;
199            }
200            std::collections::hash_map::Entry::Vacant(slot) => {
201                slot.insert(self.entries.len());
202                self.entries.push((row, col, text));
203            }
204        }
205    }
206
207    fn remove(&mut self, row: u32, col: u32) -> Option<String> {
208        let deferred_text = self.get(row, col);
209        self.invalidate_deferred_at(row, col);
210        let Some(idx) = self.index.remove(&(row, col)) else {
211            return deferred_text;
212        };
213        let (_, _, text) = self.entries.remove(idx);
214        // `Vec::remove` shifted everything after `idx` left by one.
215        for slot in self.index.values_mut() {
216            if *slot > idx {
217                *slot -= 1;
218            }
219        }
220        Some(text)
221    }
222
223    fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
224        self.index
225            .get(&(row, col))
226            .map(|&i| self.entries[i].2.as_str())
227    }
228
229    fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
230        let idx = self.index.remove(&(row, col))?;
231        let (_, _, text) = self.entries.remove(idx);
232        for slot in self.index.values_mut() {
233            if *slot > idx {
234                *slot -= 1;
235            }
236        }
237        Some(text)
238    }
239
240    fn get(&self, row: u32, col: u32) -> Option<String> {
241        if let Some(text) = self.get_ordinary(row, col) {
242            return Some(text.to_string());
243        }
244        let package = self.deferred_package.as_ref()?;
245        if package.suppressed.contains(&(row, col)) {
246            return None;
247        }
248        package
249            .replay
250            .lock()
251            .ok()?
252            .formula_at(row, col)
253            .ok()?
254            .map(|record| record.text)
255    }
256
257    fn len(&self) -> usize {
258        self.deferred_package
259            .as_ref()
260            .map_or(self.entries.len(), |package| {
261                (if package.source_geometry_complete
262                    && (package.coordinates_cover_families
263                        || (package.families.is_empty() && package.partitioned_families.is_empty()))
264                {
265                    package.source_coordinates.len()
266                } else {
267                    usize::try_from(package.report.source_formula_records_spooled)
268                        .unwrap_or(usize::MAX)
269                })
270                .saturating_sub(package.suppressed.len())
271                .saturating_add(self.entries.len())
272            })
273    }
274
275    fn is_empty(&self) -> bool {
276        self.entries.is_empty() && self.deferred_package.is_none()
277    }
278
279    fn into_parts(self) -> StagedSheetParts {
280        (self.entries, self.deferred_package)
281    }
282}
283
284type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
285
286#[derive(Clone, Debug, PartialEq, Eq, Hash)]
287struct PreparationRegion {
288    sheet: String,
289    sheet_id: SheetId,
290    start_row: u32,
291    start_col: u32,
292    end_row: u32,
293    end_col: u32,
294}
295
296// Target roots dedupe by identity; they keep their first-seen order.
297struct OrderedTargetProducers {
298    ordered: Vec<crate::engine::target_preparation::TargetProducer>,
299    seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
300}
301
302impl OrderedTargetProducers {
303    fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
304        let mut ordered = Vec::new();
305        ordered.try_reserve(capacity)?;
306        let mut seen = FxHashSet::default();
307        seen.try_reserve(capacity)?;
308        Ok(Self { ordered, seen })
309    }
310
311    fn from_ordered(
312        ordered: Vec<crate::engine::target_preparation::TargetProducer>,
313    ) -> Result<Self, std::collections::TryReserveError> {
314        let mut seen = FxHashSet::default();
315        seen.try_reserve(ordered.len())?;
316        seen.extend(ordered.iter().copied());
317        Ok(Self { ordered, seen })
318    }
319
320    fn push(
321        &mut self,
322        producer: crate::engine::target_preparation::TargetProducer,
323    ) -> Result<bool, std::collections::TryReserveError> {
324        #[cfg(test)]
325        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
326        if self.seen.contains(&producer) {
327            return Ok(false);
328        }
329        self.seen.try_reserve(1)?;
330        self.ordered.try_reserve(1)?;
331        self.seen.insert(producer);
332        self.ordered.push(producer);
333        Ok(true)
334    }
335
336    fn len(&self) -> usize {
337        self.ordered.len()
338    }
339
340    fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
341        self.ordered
342    }
343}
344
345#[cfg(test)]
346thread_local! {
347    static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
348}
349
350fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
351    crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
352        reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
353        limit: u64::MAX,
354        observed: observed as u64,
355        request_id,
356    })
357    .into_excel_error()
358}
359
360#[derive(Debug)]
361struct PreparedOrdinaryStagedFormula {
362    sheet: String,
363    sheet_id: SheetId,
364    lease: StagedFormulaLease,
365    ast_id: Option<AstNodeId>,
366    plan: Option<DependencyPlanRow>,
367}
368
369struct PreparedTargetSourcePackage {
370    sheet: String,
371    sheet_id: SheetId,
372    lease: StagedPackageLease,
373    selected_points: Option<BTreeSet<(u32, u32)>>,
374    complete_selections: BTreeSet<crate::engine::SourceFamilyId>,
375    deferred_shared: bool,
376    direct_domains: Vec<(
377        crate::engine::SourceFamilyId,
378        crate::engine::PlacementDomainTransport,
379    )>,
380    source_report: crate::engine::FormulaCompressedSourceReport,
381    replay_records: Vec<crate::engine::DeferredReplayFormula>,
382    spool_replays: u64,
383    disposition: crate::engine::FormulaReplayDisposition,
384    legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
385    direct_families: usize,
386    direct_cells: u64,
387    direct_fragments: u64,
388    direct_complete_families: u64,
389    direct_complete_cells: u64,
390    direct_partition_families: u64,
391    direct_partition_cells: u64,
392    anchor_parses: u64,
393    anchor_asts: u64,
394    anchor_analyses: u64,
395}
396
397impl PreparedTargetSourcePackage {
398    fn empty_selection(sheet: &str, sheet_id: SheetId, lease: StagedPackageLease) -> Self {
399        Self {
400            sheet: sheet.to_owned(),
401            sheet_id,
402            lease,
403            selected_points: Some(BTreeSet::new()),
404            complete_selections: Default::default(),
405            deferred_shared: false,
406            direct_domains: Vec::new(),
407            source_report: Default::default(),
408            replay_records: Vec::new(),
409            spool_replays: 0,
410            disposition: Default::default(),
411            legacy: Vec::new(),
412            direct_families: 0,
413            direct_cells: 0,
414            direct_fragments: 0,
415            direct_complete_families: 0,
416            direct_complete_cells: 0,
417            direct_partition_families: 0,
418            direct_partition_cells: 0,
419            anchor_parses: 0,
420            anchor_asts: 0,
421            anchor_analyses: 0,
422        }
423    }
424
425    fn direct_contains(&self, row: u32, col: u32) -> bool {
426        self.direct_domains.iter().any(|(_, domain)| {
427            let rect = domain.rect();
428            row > rect.start.row
429                && row <= rect.end.row + 1
430                && col > rect.start.col
431                && col <= rect.end.col + 1
432        })
433    }
434
435    fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
436        self.replay_records.iter().filter(|record| {
437            let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
438                return true;
439            };
440            let coord = crate::engine::SourceCoord { row, col };
441            let disposition = match record.family {
442                Some(family) => self.disposition.shared_disposition(family, coord),
443                None => self.disposition.ordinary_disposition(coord).0,
444            };
445            !matches!(
446                disposition,
447                crate::engine::FormulaReplayCoordinateDisposition::Direct
448                    | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
449            )
450        })
451    }
452}
453
454type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
455type StagedFormulaBatches = Vec<(String, StagedSheet)>;
456
457type CompressedReplayBatch = (
458    FormulaIngestBatch,
459    crate::engine::FormulaCompressedSourceBatch,
460);
461type PreparedSourceBatch = (
462    FormulaIngestBatch,
463    crate::engine::FormulaCompressedSourceReport,
464    crate::engine::FormulaCompressedPreparation,
465);
466type PreparedStagedFormulaBatches = (
467    PreparedFormulaBatches,
468    Vec<CompressedReplayBatch>,
469    Vec<PreparedSourceBatch>,
470    // Formulas whose planning can fail (see `formula_may_fail_planning`).
471    FxHashSet<crate::engine::arena::AstNodeId>,
472);
473
474/// Backend-neutral source-family ingress. Adapters may prepare candidates and
475/// submit exact replay; every formula is still materialized per cell.
476#[doc(hidden)]
477pub struct SourceFormulaIngress<'a, R> {
478    engine: &'a mut Engine<R>,
479}
480
481impl<R> SourceFormulaIngress<'_, R>
482where
483    R: EvaluationContext,
484{
485    pub fn prepare_families(
486        &mut self,
487        sheet_name: &str,
488        families: &[crate::engine::SourceFormulaFamily],
489    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
490        self.engine.observe_function_semantic_epoch()?;
491        Ok(self
492            .engine
493            .prepare_source_formula_families(sheet_name, families))
494    }
495
496    pub fn prepare_eager_proposals(
497        &mut self,
498        sheet_name: &str,
499        families: &[crate::engine::SourceFormulaFamily],
500        partitions: &[crate::engine::PartitionedSourceFormulaFamily],
501        formula_record_count: u64,
502        replay: Box<dyn crate::engine::DeferredFormulaReplay>,
503    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
504        self.engine.observe_function_semantic_epoch()?;
505        let replay = Arc::new(std::sync::Mutex::new(replay));
506        self.engine.prepare_source_formula_proposals(
507            sheet_name,
508            families,
509            partitions,
510            partitions,
511            formula_record_count,
512            replay,
513            &BTreeSet::new(),
514            &Default::default(),
515            None,
516        )
517    }
518
519    pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
520        self.engine.stage_deferred_formula_package(package);
521    }
522
523    pub fn ingest_replay_batches(
524        &mut self,
525        batches: Vec<(
526            FormulaIngestBatch,
527            crate::engine::FormulaCompressedSourceBatch,
528        )>,
529    ) -> Result<FormulaIngestReport, ExcelError> {
530        self.engine
531            .ingest_compressed_formula_source_batches(batches)
532    }
533
534    pub fn finish_prepared(
535        &mut self,
536        batches: Vec<(
537            FormulaIngestBatch,
538            crate::engine::FormulaCompressedSourceReport,
539            crate::engine::FormulaCompressedPreparation,
540        )>,
541    ) -> Result<FormulaIngestReport, ExcelError> {
542        self.engine.finish_compressed_formula_sources(batches)
543    }
544}
545
546// Computed-write coalescing pays a fixed grouping/planning cost. For very narrow
547// layers there is not enough work to amortize it, and the direct point-write path
548// is faster while preserving the same visibility semantics.
549const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
550
551/// Equal values, numbers by bits (debug oracles).
552#[cfg(debug_assertions)]
553fn same_value_bits(a: &LiteralValue, b: &LiteralValue) -> bool {
554    match (a, b) {
555        (LiteralValue::Number(x), LiteralValue::Number(y)) => x.to_bits() == y.to_bits(),
556        _ => a == b,
557    }
558}
559
560/// Whether a layer's computed writes are buffered: not for a chain unit,
561/// whose members read the ones written before them.
562fn buffer_layer_writes(layer: &crate::engine::scheduler::Layer) -> bool {
563    !layer.sequential && layer.vertices.len() >= COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH
564}
565/// Adaptive layer parallelism (see `evaluate_layer_parallel`): a layer runs
566/// sequentially until either it has run `PARALLEL_LAYER_PROBE` or the rest,
567/// at the rate so far, is estimated at `PARALLEL_LAYER_WORTH` or more; then
568/// the rest goes to the thread pool. Tuned on Enron first eval (probe 30 /
569/// 100 / 300 µs, worth 50 / 150 / 400 µs; 300 / 150 best, no workbook
570/// slower).
571/// Candidate count from which schedule preparation sorts on the pool.
572const PARALLEL_SCHEDULE_MIN_CANDIDATES: usize = 16 * 1024;
573/// Plan reuse: a schedule of more candidates than this can become the
574/// base, and a request takes the base restricted to it when the base is at
575/// most `BASE_SCHEDULE_RATIO` times the request.
576const BASE_SCHEDULE_MIN_VERTICES: usize = 64;
577const BASE_SCHEDULE_RATIO: usize = 256;
578/// A restricted schedule keeps the base's layers, which can split a family
579/// run the planner would keep whole (each piece then builds its own
580/// criteria index): below this many candidates planning costs less than
581/// that (real_ops_model: 75 candidates, 31 us to plan, +0.3 ms restricted).
582const BASE_SCHEDULE_MIN_REQUEST: usize = 128;
583const PARALLEL_LAYER_PROBE: std::time::Duration = std::time::Duration::from_micros(300);
584const PARALLEL_LAYER_WORTH: std::time::Duration = std::time::Duration::from_micros(150);
585/// A member this expensive (ns, measured by the probe) is its own task.
586const EXPENSIVE_VERTEX_NS: u128 = 10_000;
587/// [`Engine::live_cancellation_after_work`] messages.
588const UNIT_CANCELLED: &str = "Evaluation cancelled; the evaluated unit was not committed";
589const GROUP_CANCELLED: &str =
590    "Parallel evaluation cancelled; the evaluated group was not committed";
591const SCC_CANCELLED: &str = "Evaluation cancelled; the evaluated cycle was not completed";
592
593#[derive(Debug, Clone, PartialEq)]
594pub(crate) enum ComputedWrite {
595    Cell {
596        seq: u64,
597        sheet_id: SheetId,
598        row0: u32,
599        col0: u32,
600        value: OverlayValue,
601        format_id: Option<crate::format::FormatId>,
602    },
603    Rect {
604        seq: u64,
605        sheet_id: SheetId,
606        sr0: u32,
607        sc0: u32,
608        values: Vec<Vec<OverlayValue>>,
609    },
610    /// Consecutive rows `row0..` of one column (a family run's commit),
611    /// each with its format.
612    Run {
613        seq: u64,
614        sheet_id: SheetId,
615        row0: u32,
616        col0: u32,
617        entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
618    },
619}
620
621impl ComputedWrite {
622    #[inline]
623    pub(crate) fn seq(&self) -> u64 {
624        match self {
625            ComputedWrite::Cell { seq, .. }
626            | ComputedWrite::Rect { seq, .. }
627            | ComputedWrite::Run { seq, .. } => *seq,
628        }
629    }
630}
631
632#[derive(Debug, Default)]
633pub(crate) struct ComputedWriteBuffer {
634    writes: Vec<ComputedWrite>,
635    next_seq: u64,
636    estimated_bytes: usize,
637    formats_present: bool,
638}
639
640impl ComputedWriteBuffer {
641    const ENTRY_BASE_BYTES: usize = 32;
642
643    #[inline]
644    pub(crate) fn is_empty(&self) -> bool {
645        self.writes.is_empty()
646    }
647
648    #[inline]
649    pub(crate) fn len(&self) -> usize {
650        self.writes.len()
651    }
652
653    #[inline]
654    pub(crate) fn writes(&self) -> &[ComputedWrite] {
655        &self.writes
656    }
657
658    #[inline]
659    pub(crate) fn estimated_bytes(&self) -> usize {
660        self.estimated_bytes
661    }
662
663    #[cfg(feature = "tracing")]
664    fn traced_cell_count(&self) -> usize {
665        self.writes
666            .iter()
667            .map(|write| match write {
668                ComputedWrite::Cell { .. } => 1,
669                ComputedWrite::Rect { values, .. } => values.iter().map(Vec::len).sum(),
670                ComputedWrite::Run { entries, .. } => entries.len(),
671            })
672            .sum()
673    }
674
675    pub(crate) fn push_cell(
676        &mut self,
677        sheet_id: SheetId,
678        row0: u32,
679        col0: u32,
680        value: OverlayValue,
681    ) {
682        self.push_cell_with_format(sheet_id, row0, col0, value, None);
683    }
684
685    pub(crate) fn push_cell_with_format(
686        &mut self,
687        sheet_id: SheetId,
688        row0: u32,
689        col0: u32,
690        value: OverlayValue,
691        format_id: Option<crate::format::FormatId>,
692    ) {
693        let format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
694        self.formats_present |= format_id.is_some();
695        let seq = self.next_sequence();
696        self.estimated_bytes = self
697            .estimated_bytes
698            .saturating_add(Self::estimate_value_bytes(&value));
699        self.writes.push(ComputedWrite::Cell {
700            seq,
701            sheet_id,
702            row0,
703            col0,
704            value,
705            format_id,
706        });
707    }
708
709    pub(crate) fn push_column_run(
710        &mut self,
711        sheet_id: SheetId,
712        row0: u32,
713        col0: u32,
714        mut entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
715    ) {
716        let seq = self.next_sequence();
717        let mut added = 0usize;
718        for (value, format_id) in entries.iter_mut() {
719            *format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
720            self.formats_present |= format_id.is_some();
721            added = added.saturating_add(Self::estimate_value_bytes(value));
722        }
723        self.estimated_bytes = self.estimated_bytes.saturating_add(added);
724        self.writes.push(ComputedWrite::Run {
725            seq,
726            sheet_id,
727            row0,
728            col0,
729            entries,
730        });
731    }
732
733    pub(crate) fn push_rect(
734        &mut self,
735        sheet_id: SheetId,
736        sr0: u32,
737        sc0: u32,
738        values: Vec<Vec<OverlayValue>>,
739    ) {
740        let seq = self.next_sequence();
741        let added = values
742            .iter()
743            .flat_map(|row| row.iter())
744            .map(Self::estimate_value_bytes)
745            .fold(0usize, usize::saturating_add);
746        self.estimated_bytes = self.estimated_bytes.saturating_add(added);
747        self.writes.push(ComputedWrite::Rect {
748            seq,
749            sheet_id,
750            sr0,
751            sc0,
752            values,
753        });
754    }
755
756    pub(crate) fn clear(&mut self) {
757        self.writes.clear();
758        self.estimated_bytes = 0;
759        self.formats_present = false;
760    }
761
762    fn take_writes(&mut self) -> (Vec<ComputedWrite>, bool) {
763        self.estimated_bytes = 0;
764        let formats_present = std::mem::take(&mut self.formats_present);
765        (std::mem::take(&mut self.writes), formats_present)
766    }
767
768    fn next_sequence(&mut self) -> u64 {
769        let seq = self.next_seq;
770        self.next_seq = self.next_seq.wrapping_add(1);
771        seq
772    }
773
774    #[inline]
775    fn estimate_value_bytes(value: &OverlayValue) -> usize {
776        Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
777    }
778}
779
780#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
781struct ComputedWriteChunkKey {
782    sheet_id: SheetId,
783    col0: u32,
784    chunk_idx: usize,
785    chunk_start_row0: u32,
786}
787
788#[derive(Debug, Clone, PartialEq)]
789pub(crate) struct ComputedWriteChunkEntryPlan {
790    pub(crate) row_in_chunk: usize,
791    pub(crate) seq: u64,
792    pub(crate) value: OverlayValue,
793    pub(crate) format_id: Option<crate::format::FormatId>,
794}
795
796#[derive(Debug, Clone, PartialEq, Eq)]
797pub(crate) enum ComputedWriteChunkPlanShape {
798    Point,
799    SparseOffsets {
800        entries: usize,
801        span_len: usize,
802    },
803    DenseRange {
804        start: usize,
805        len: usize,
806    },
807    RunRange {
808        start: usize,
809        len: usize,
810        runs: usize,
811    },
812}
813
814#[derive(Debug, Clone, PartialEq, Eq)]
815pub(crate) enum ComputedWriteFormatClear {
816    Range { start: usize, end: usize },
817    Offsets(Vec<usize>),
818}
819
820#[derive(Debug, Clone, PartialEq)]
821pub(crate) enum ComputedWriteChunkFormatEffect {
822    NoFormatWork,
823    ClearStale(ComputedWriteFormatClear),
824    SetExplicit(Vec<(usize, Option<crate::format::FormatId>)>),
825}
826
827#[derive(Debug, Clone, PartialEq)]
828pub(crate) struct ComputedWriteChunkPlan {
829    pub(crate) sheet_id: SheetId,
830    pub(crate) col0: u32,
831    pub(crate) chunk_idx: usize,
832    pub(crate) chunk_start_row0: u32,
833    pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
834    pub(crate) shape: ComputedWriteChunkPlanShape,
835    pub(crate) format_effect: ComputedWriteChunkFormatEffect,
836}
837
838#[derive(Debug, Clone, Default, PartialEq)]
839pub(crate) struct ComputedWriteCoalescingPlan {
840    pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
841    pub(crate) input_cells: usize,
842    pub(crate) coalesced_cells: usize,
843    pub(crate) overwritten_cells: usize,
844}
845
846impl ComputedWriteCoalescingPlan {
847    #[inline]
848    pub(crate) fn is_empty(&self) -> bool {
849        self.chunks.is_empty()
850    }
851}
852
853impl ComputedWriteChunkPlan {
854    fn from_group(
855        key: ComputedWriteChunkKey,
856        mut entries: Vec<ComputedWriteChunkEntryPlan>,
857        formats_present: bool,
858        computed_lane_has_formats: bool,
859    ) -> (Self, usize) {
860        entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
861        let input_len = entries.len();
862        let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
863        for entry in entries {
864            if let Some(prev) = coalesced.last_mut()
865                && prev.row_in_chunk == entry.row_in_chunk
866            {
867                *prev = entry;
868                continue;
869            }
870            coalesced.push(entry);
871        }
872        let overwritten = input_len.saturating_sub(coalesced.len());
873        let shape = Self::classify_shape(&coalesced);
874        let format_effect = Self::classify_format_effect(
875            &coalesced,
876            &shape,
877            formats_present,
878            computed_lane_has_formats,
879        );
880        (
881            Self {
882                sheet_id: key.sheet_id,
883                col0: key.col0,
884                chunk_idx: key.chunk_idx,
885                chunk_start_row0: key.chunk_start_row0,
886                entries: coalesced,
887                shape,
888                format_effect,
889            },
890            overwritten,
891        )
892    }
893
894    fn classify_format_effect(
895        entries: &[ComputedWriteChunkEntryPlan],
896        shape: &ComputedWriteChunkPlanShape,
897        formats_present: bool,
898        computed_lane_has_formats: bool,
899    ) -> ComputedWriteChunkFormatEffect {
900        if !formats_present {
901            return if computed_lane_has_formats {
902                ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
903            } else {
904                ComputedWriteChunkFormatEffect::NoFormatWork
905            };
906        }
907
908        if entries.iter().all(|entry| entry.format_id.is_none()) {
909            return if computed_lane_has_formats {
910                ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
911            } else {
912                ComputedWriteChunkFormatEffect::NoFormatWork
913            };
914        }
915
916        ComputedWriteChunkFormatEffect::SetExplicit(
917            entries
918                .iter()
919                .map(|entry| (entry.row_in_chunk, entry.format_id))
920                .collect(),
921        )
922    }
923
924    fn format_clear_spec(
925        entries: &[ComputedWriteChunkEntryPlan],
926        shape: &ComputedWriteChunkPlanShape,
927    ) -> ComputedWriteFormatClear {
928        match shape {
929            ComputedWriteChunkPlanShape::DenseRange { start, len }
930            | ComputedWriteChunkPlanShape::RunRange { start, len, .. } => {
931                ComputedWriteFormatClear::Range {
932                    start: *start,
933                    end: start.saturating_add(*len),
934                }
935            }
936            ComputedWriteChunkPlanShape::Point
937            | ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
938                ComputedWriteFormatClear::Offsets(
939                    entries.iter().map(|entry| entry.row_in_chunk).collect(),
940                )
941            }
942        }
943    }
944
945    fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
946        debug_assert!(!entries.is_empty());
947        if entries.len() == 1 {
948            return ComputedWriteChunkPlanShape::Point;
949        }
950
951        let start = entries[0].row_in_chunk;
952        let end = entries[entries.len() - 1].row_in_chunk;
953        let span_len = end.saturating_sub(start).saturating_add(1);
954        if span_len != entries.len() {
955            return ComputedWriteChunkPlanShape::SparseOffsets {
956                entries: entries.len(),
957                span_len,
958            };
959        }
960
961        let runs = Self::run_count(entries);
962        if runs < entries.len() {
963            ComputedWriteChunkPlanShape::RunRange {
964                start,
965                len: entries.len(),
966                runs,
967            }
968        } else {
969            ComputedWriteChunkPlanShape::DenseRange {
970                start,
971                len: entries.len(),
972            }
973        }
974    }
975
976    fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
977        let mut runs = 0usize;
978        let mut prev: Option<&OverlayValue> = None;
979        for entry in entries {
980            if prev != Some(&entry.value) {
981                runs = runs.saturating_add(1);
982                prev = Some(&entry.value);
983            }
984        }
985        runs
986    }
987}
988
989#[cfg(feature = "tracing")]
990#[derive(Default)]
991struct TraceEvaluationCounters {
992    computed_vertices: usize,
993    cycles: usize,
994}
995
996pub struct Engine<R> {
997    criteria_mask_cache: criteria_mask_cache::CriteriaMaskCache,
998    pub(crate) graph: DependencyGraph,
999    resolver: R,
1000    pub(super) resolved_functions: crate::interpreter::ResolvedFunctions,
1001    resolved_sheet_names:
1002        FxHashMap<crate::engine::arena::StringId, (crate::SheetId, Option<usize>)>,
1003    resolution_sheet_revision: Option<(u64, u64, usize)>,
1004    resolution_arrow_sheet_names: Vec<Arc<str>>,
1005    resolved_arrow_sheets: Vec<Option<usize>>,
1006    /// Stable-ID index, invalidated before exposing arbitrary mutable storage.
1007    arrow_sheet_index: std::sync::OnceLock<Vec<Option<usize>>>,
1008    pub config: EvalConfig,
1009    workbook_load_limits: crate::engine::WorkbookLoadLimits,
1010    /// Clock for volatile date/time builtins, wrapped in a per-recalc
1011    /// snapshot: sampled once at the start of every evaluation request
1012    /// ([`Self::begin_evaluation_request`]) so all `NOW()`/`TODAY()` reads in
1013    /// one recalc — including SCC iteration passes — agree (spec §7.11).
1014    clock: crate::timezone::SnapshotClock,
1015    thread_pool: Option<Arc<rayon::ThreadPool>>,
1016    pub recalc_epoch: u64,
1017    snapshot_id: std::sync::atomic::AtomicU64,
1018    topology_epoch: u64,
1019    cached_static_schedule: Option<CachedScheduleEntry>,
1020    /// Program 3 (plan reuse): schedules of recent earlier requests, most
1021    /// recent last (a user alternating between a few inputs recalculates
1022    /// the same few closures). Bounded by `RECENT_SCHEDULES` entries and
1023    /// `RECENT_SCHEDULE_VERTICES` candidate vertices in total.
1024    recent_schedules: Vec<CachedScheduleEntry>,
1025    /// Program 3 (plan reuse): the largest current schedule seen (usually
1026    /// the first evaluation's). A request it covers, and that is not much
1027    /// smaller, takes its schedule restricted to the request instead of
1028    /// planning (`Schedule::restrict`).
1029    base_schedule: Option<CachedScheduleEntry>,
1030    #[cfg(any(test, feature = "benchmark_internal"))]
1031    recalc_reuse_probe: std::sync::Mutex<RecalcReuseProbe>,
1032    spill_mgr: ShimSpillManager,
1033    /// Arrow-backed storage for sheet values (Phase A)
1034    arrow_sheets: SheetStore,
1035    /// Workbook-local number-format registry.
1036    format_registry: crate::format::FormatRegistry,
1037    /// Derived formula formats keyed by grid position, never graph vertex identity.
1038    derived_formats: crate::engine::derived_formats::DerivedFormats,
1039    #[cfg(test)]
1040    derived_format_operations_for_test: std::sync::atomic::AtomicU64,
1041    #[cfg(test)]
1042    family_members_for_test: std::sync::atomic::AtomicU64,
1043    #[cfg(test)]
1044    invariant_bound_members_for_test: std::sync::atomic::AtomicU64,
1045    #[cfg(test)]
1046    lifted_members_for_test: std::sync::atomic::AtomicU64,
1047    #[cfg(test)]
1048    chained_members_for_test: std::sync::atomic::AtomicU64,
1049    #[cfg(test)]
1050    lane_clean_reads_for_test: std::sync::atomic::AtomicU64,
1051    #[cfg(test)]
1052    criteria_kernel_members_for_test: std::sync::atomic::AtomicU64,
1053    #[cfg(test)]
1054    memo_hits_for_test: std::sync::atomic::AtomicU64,
1055    /// Authority build last compressed (`maybe_compress_formulas`).
1056    compressed_at_build: Option<u64>,
1057    #[cfg(test)]
1058    computed_overlay_set_explicit_entry_operations_for_test: u64,
1059    #[cfg(test)]
1060    computed_overlay_stale_clear_range_effects_for_test: u64,
1061    #[cfg(test)]
1062    computed_overlay_stale_clear_offset_attempts_for_test: u64,
1063    #[cfg(test)]
1064    computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64,
1065    /// True if any edit after bulk load; disables Arrow reads for parity
1066    has_edited: bool,
1067    /// Overlay compaction counter (Phase C instrumentation)
1068    overlay_compactions: u64,
1069
1070    // Overlay memory observability / budget (ticket 503)
1071    computed_overlay_bytes_estimate: usize,
1072    computed_overlay_mirroring_disabled: bool,
1073    /// When true, RangeView resolution materializes from graph/Arrow base per-cell.
1074    /// This preserves correctness if we stop mirroring formula/spill outputs into computed overlays.
1075    pub(crate) force_materialize_range_views: bool,
1076    // Pass-scoped cache for Arrow used-row bounds per column
1077    row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
1078    // Snapshot-scoped final used-axis bounds for open-ended references.
1079    used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
1080    lookup_index_cache: LookupIndexCache,
1081    source_cache: Arc<std::sync::RwLock<SourceCache>>,
1082    /// Identity binding for opaque source-family preparations.
1083    source_formula_token: Arc<()>,
1084    /// Load-time staging of relocated formula copies (see `SourceFamilyMode`).
1085    pub(crate) source_family_mode: crate::engine::SourceFamilyMode,
1086    pub(crate) source_family_counters: crate::engine::SourceFamilyCounters,
1087    /// Dedicated identity binding for reusable recalculation plans.
1088    recalc_plan_token: Arc<()>,
1089    /// Staged formulas by sheet when `defer_graph_building` is enabled.
1090    staged_formulas: StagedFormulaMap,
1091    /// Presence and generation authority for ordinary staged formula discovery.
1092    staged_formula_index: StagedFormulaIndex,
1093    // Occupancy invalidation only: never a formula/read dependency.
1094    blocked_pending_spills: Vec<(VertexId, CellRef, Region)>,
1095    /// Anchors whose spill is blocked by a formula or by another spill's
1096    /// cell, keyed by that blocking cell: a change to the cell (an edit, or
1097    /// the other spill clearing) wakes them. Never a read dependency.
1098    spill_blocker_waiters: SpillBlockerWaiters,
1099    /// Per-sheet row visibility sidecar state.
1100    row_visibility: FxHashMap<SheetId, RowVisibilityState>,
1101    /// Cached row visibility masks keyed by sheet/span/mode/version.
1102    row_visibility_mask_cache: std::sync::RwLock<
1103        FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
1104    >,
1105    /// Non-fatal malformed formula diagnostics captured during ingest/graph-build.
1106    formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
1107    /// Last centralized formula ingest report.
1108    last_formula_ingest_report: Option<FormulaIngestReport>,
1109    /// Aggregate centralized formula ingest report for this engine.
1110    formula_ingest_report_total: FormulaIngestReport,
1111    /// Transient cancellation flag used during evaluation
1112    active_cancel_flag: Option<crate::engine::CancelToken>,
1113    /// Transient absolute deadline used by composed target and plan requests.
1114    active_evaluation_deadline: Option<Instant>,
1115
1116    /// Engine-level action depth.
1117    ///
1118    /// Ticket 614 introduces `Engine::action` as a stable, commit-only transaction surface.
1119    /// Nested actions are currently disallowed (deterministic rule) and will return an error.
1120    action_depth: u32,
1121
1122    // Phase 3b virtual-dependency convergence telemetry
1123    last_virtual_dep_telemetry: VirtualDepTelemetry,
1124    virtual_dep_fallback_activations: u64,
1125
1126    // Runtime-cycle SCC evaluation telemetry (RFC #112, Stage 2)
1127    last_cycle_telemetry: CycleTelemetry,
1128
1129    // C0 evaluation-resource observability. IDs are never reset or reused.
1130    next_evaluation_resource_request_id: u64,
1131    evaluation_resource_request_depth: usize,
1132    active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1133    last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1134    evaluation_resource_baseline: EvaluationResourceBaselineStats,
1135    evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1136    evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1137    evaluation_resource_config_diagnostic:
1138        Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1139    active_resource_ledger: Option<ResourceLedger>,
1140    source_cache_footprints: Vec<std::sync::Weak<std::sync::atomic::AtomicU64>>,
1141    source_cache_accounted: u64,
1142
1143    /// SCC members that entered iterative calculation (`CyclePolicy::Iterate`
1144    /// with a witnessed live cycle) during the current evaluation request
1145    /// and must re-run on the next one.
1146    ///
1147    /// Excel re-evaluates circular cells on EVERY recalc (the accumulator
1148    /// contract, spec §4/§7.6), but this engine's dirty model marks SCC
1149    /// members clean after a recalc and would otherwise skip them forever.
1150    /// Resolution: members of iterating SCCs are redirtied volatile-like at
1151    /// the end of the same recalc that iterated them
1152    /// ([`Self::redirty_for_next_recalc`], called wherever
1153    /// `redirty_volatiles` runs). The set is per-recalc, never persisted:
1154    /// if an edit breaks the cycle, the next recalc's SCC task either does
1155    /// not exist or settles as phantom, nothing re-registers, and the
1156    /// redirty chain stops by itself.
1157    ///
1158    /// SCCs that landed on an exact fixed point are exempt: they go to
1159    /// [`Self::retained_scc_members`] instead and stay clean until the dirty
1160    /// graph (or a config change) reaches them (#368).
1161    pending_iterative_redirty: Vec<VertexId>,
1162    /// Opt-in snapshot request: defer only next-cycle volatile redirty.
1163    snapshot_evaluation_active: bool,
1164    snapshot_volatile_redirty_pending: bool,
1165    /// Members of iterating SCCs retained across recalcs (#368), keyed to the
1166    /// id of the retained SCC they belong to (ids come from
1167    /// `next_retained_scc_id`; grouping is only used for telemetry).
1168    ///
1169    /// An SCC is retained when the recalc that iterated it stopped because
1170    /// every member reproduced its previous value exactly (|Δ| = 0, or
1171    /// identity for non-numeric members; never NaN-converged), before the
1172    /// `max_iterations` cap, with no volatile or dynamic-reference member.
1173    /// Such an SCC is a fixed point of its own inputs: running it again with
1174    /// the same inputs cannot change any value, so it is not redirtied. The
1175    /// dirty graph remains the validity authority — any edit that reaches a
1176    /// member dirties it like any other formula and the SCC task re-runs.
1177    /// Membership is dropped when a member runs in an SCC task again (it is
1178    /// then re-retained or re-registered for per-recalc redirty), when the
1179    /// vertex is deleted, or when
1180    /// [`Self::reconcile_retained_sccs_at_request_begin`] invalidates the
1181    /// whole set because a config knob outside the graph changed.
1182    retained_scc_members: FxHashMap<VertexId, u64>,
1183    next_retained_scc_id: u64,
1184    /// [`Self::retained_scc_config_fingerprint`] as of the last retention.
1185    /// `Engine::config` is a public field, so knobs that change a retained
1186    /// SCC's result (cycle policy/tolerance, date system, determinism,
1187    /// volatile seeding) can change between recalcs without touching the
1188    /// graph; a mismatch at request begin dirties every retained member.
1189    /// Only meaningful while `retained_scc_members` is non-empty.
1190    retained_scc_config_fingerprint: u64,
1191    /// Function-registry semantic epoch and runtime-provider revision as of
1192    /// the last time retained SCCs were reconciled against them. A newer
1193    /// epoch dirties only the retained members whose formula calls a changed
1194    /// function (or every member when the change log is incomplete).
1195    retained_scc_function_epoch_seen: u64,
1196    retained_scc_provider_revision_seen: Option<u64>,
1197    /// Retained members that were already dirty when the current request
1198    /// began, with the SCC id they carried. A member still holding that id
1199    /// at the end of the request was not touched by any SCC task — its
1200    /// cycle dissolved (it evaluated as an ordinary formula) or the request
1201    /// never reached it — so [`Self::redirty_for_next_recalc`] drops it from
1202    /// the retained set instead of letting it linger.
1203    retained_scc_dirty_at_begin: Vec<(VertexId, u64)>,
1204
1205    /// Final committed values of iterating-SCC members (spec §4 persistence).
1206    /// In canonical (value-cache disabled) mode the computed overlay is the
1207    /// ONLY home of a formula's value, and structural edits clear computed
1208    /// overlays wholesale (`clear_computed_overlay_after_row/_col`) —
1209    /// destroying iteration state (accumulators reset to 0; found by the
1210    /// iterate edge corpus). This snapshot lets the next SCC task re-seed
1211    /// members whose overlay entry vanished. Members registered for
1212    /// per-recalc redirty are refreshed by
1213    /// [`Self::redirty_for_next_recalc`]; retained members are written once
1214    /// when retained. Entries are dropped when the member's SCC task ends
1215    /// without iterating or when the vertex is deleted. Empty unless
1216    /// something iterated — zero cost otherwise.
1217    iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1218
1219    /// Global function-registry semantic epoch last observed.
1220    function_semantic_epoch_seen: u64,
1221    /// Runtime-provider semantic revision last observed.
1222    function_provider_revision_seen: Option<u64>,
1223
1224    #[cfg(feature = "tracing")]
1225    trace_evaluation_counters: TraceEvaluationCounters,
1226    #[cfg(test)]
1227    evaluation_request_begin_count_for_test: u64,
1228    #[cfg(any(test, feature = "test-support"))]
1229    before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1230    #[cfg(test)]
1231    before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1232    #[cfg(test)]
1233    before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1234    #[cfg(test)]
1235    inject_target_semantic_stale_once_for_test: bool,
1236    #[cfg(test)]
1237    force_virtual_dep_changes_remaining_for_test: usize,
1238    /// Dynamic-read freshness state (design §8.2).
1239    freshness: freshness::Freshness,
1240    #[cfg(test)]
1241    fail_evaluation_commit_preflight_once_for_test: bool,
1242    #[cfg(test)]
1243    target_preparation_fault_for_test:
1244        Option<crate::engine::target_preparation::TargetPreparationFault>,
1245    #[cfg(test)]
1246    force_non_cycle_schedule_fallback_for_test: bool,
1247    #[cfg(test)]
1248    before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1249    #[cfg(test)]
1250    after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1251}
1252
1253/// This wrapper is intentionally thin for ticket 614 (commit-only): it delegates to existing
1254/// `Engine` edit methods and does not create changelog boundaries or implement rollback.
1255impl<R: EvaluationContext> Engine<R> {
1256    pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1257        self.graph.ingest_pipeline(&self.resolver)
1258    }
1259}
1260
1261pub struct EngineAction<'a, R>
1262where
1263    R: EvaluationContext,
1264{
1265    engine: &'a mut Engine<R>,
1266    name: String,
1267    // Complete private mutation capture used by atomic actions.
1268    // Stored as a raw pointer to avoid creating aliasing `&mut` borrows alongside `&mut Engine`.
1269    capture: Option<*mut MutationCapture>,
1270    // Optional Arrow undo journal used by `Engine::action_atomic`.
1271    // Stored as a raw pointer to avoid aliasing issues with `&mut Engine`.
1272    arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1273    // True when this EngineAction must enforce conservative atomic transaction policy.
1274    atomic_policy: bool,
1275}
1276
1277impl<'a, R> EngineAction<'a, R>
1278where
1279    R: EvaluationContext,
1280{
1281    #[inline]
1282    fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1283        let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1284        let coord = crate::reference::Coord::from_excel(row, col, true, true);
1285        crate::reference::CellRef::new(sheet_id, coord)
1286    }
1287
1288    #[inline]
1289    pub fn name(&self) -> &str {
1290        &self.name
1291    }
1292
1293    #[inline]
1294    pub fn set_cell_value(
1295        &mut self,
1296        sheet: &str,
1297        row: u32,
1298        col: u32,
1299        value: LiteralValue,
1300    ) -> Result<(), crate::engine::EditorError> {
1301        if self.capture.is_some() {
1302            let old_value = self.engine.read_cell_value(sheet, row, col);
1303            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1304            let addr = self.addr_for(sheet, row, col);
1305            let Some(capture_ptr) = self.capture else {
1306                return Err(crate::engine::EditorError::TransactionFailed {
1307                    reason: "action_with_logger: missing mutation capture".to_string(),
1308                });
1309            };
1310
1311            // For atomic journal mode, record computed overlay effects for this cell.
1312            // Delta-overlay undo is recorded semantically based on old_value/old_formula.
1313            let old_comp = if self.arrow_undo.is_some() {
1314                self.engine.read_computed_overlay_cell(sheet, row, col)
1315            } else {
1316                None
1317            };
1318
1319            if self.engine.graph_admission_enabled() {
1320                let admission =
1321                    self.engine
1322                        .graph
1323                        .preview_value_mutation(addr.sheet_id, row, col)?;
1324                self.engine.preflight_graph_admission(admission)?;
1325            }
1326            if old_formula.is_none() {
1327                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1328            }
1329
1330            let delta_old_sem = if old_formula.is_some() {
1331                None
1332            } else {
1333                Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1334            };
1335
1336            let start_len = unsafe { (&*capture_ptr).len() };
1337
1338            // Safety: `capture_ptr` comes from a unique operation-local `&mut MutationCapture`.
1339            let capture = unsafe { &mut *capture_ptr };
1340            self.engine.edit_with_capture(capture, |editor| {
1341                editor.set_cell_value_with_old_state(
1342                    addr,
1343                    value.clone(),
1344                    old_value.clone(),
1345                    old_formula.clone(),
1346                );
1347            })?;
1348            self.engine.record_structural_change(StructuralScope::Cell {
1349                sheet: addr.sheet_id,
1350                row: addr.coord.row(),
1351                col: addr.coord.col(),
1352            });
1353
1354            if let Some(undo_ptr) = self.arrow_undo {
1355                // 1) Spill snapshot operations (computed overlay rect restore).
1356                let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1357                let undo = unsafe { &mut *undo_ptr };
1358                self.engine
1359                    .record_spill_ops_into_arrow_undo(undo, new_events);
1360
1361                // 2) Delta/computed overlay single-cell deltas.
1362                let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1363                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1364                let row0 = row.saturating_sub(1);
1365                let col0 = col.saturating_sub(1);
1366                let delta_new_sem = Some(value.clone());
1367                undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1368                undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1369            }
1370            Ok(())
1371        } else {
1372            self.engine
1373                .set_cell_value(sheet, row, col, value)
1374                .map_err(crate::engine::EditorError::from)
1375        }
1376    }
1377
1378    #[inline]
1379    pub fn set_cell_formula(
1380        &mut self,
1381        sheet: &str,
1382        row: u32,
1383        col: u32,
1384        ast: ASTNode,
1385    ) -> Result<(), crate::engine::EditorError> {
1386        if self.capture.is_some() {
1387            let old_value = self.engine.read_cell_value(sheet, row, col);
1388            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1389            let addr = self.addr_for(sheet, row, col);
1390            let Some(capture_ptr) = self.capture else {
1391                return Err(crate::engine::EditorError::TransactionFailed {
1392                    reason: "action_with_logger: missing mutation capture".to_string(),
1393                });
1394            };
1395
1396            let admitted_formula = if self.engine.graph_admission_enabled() {
1397                let placement =
1398                    CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1399                let ingested = self.engine.ingest_pipeline().ingest_formula(
1400                    FormulaAstInput::Tree(ast.clone()),
1401                    placement,
1402                    None,
1403                )?;
1404                let admission = self.engine.graph.preview_formula_mutations(&[(
1405                    addr.sheet_id,
1406                    row,
1407                    col,
1408                    ingested.dep_plan.clone(),
1409                )])?;
1410                self.engine.preflight_graph_admission(admission)?;
1411                Some((ingested.ast_id, ingested.dep_plan))
1412            } else {
1413                None
1414            };
1415            if old_formula.is_none() {
1416                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1417            }
1418            let delta_old = if self.arrow_undo.is_some() {
1419                if old_formula.is_some() {
1420                    None
1421                } else {
1422                    Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1423                }
1424            } else {
1425                None
1426            };
1427            let start_len = unsafe { (&*capture_ptr).len() };
1428
1429            // Safety: `capture_ptr` comes from a unique operation-local `&mut MutationCapture`.
1430            let capture = unsafe { &mut *capture_ptr };
1431            self.engine.edit_with_capture(capture, |editor| {
1432                if let Some((ast_id, plan)) = admitted_formula {
1433                    editor.set_cell_formula_with_prepared_plan(
1434                        addr,
1435                        ast.clone(),
1436                        old_value,
1437                        old_formula,
1438                        ast_id,
1439                        plan,
1440                    );
1441                } else {
1442                    editor.set_cell_formula_with_old_state(
1443                        addr,
1444                        ast.clone(),
1445                        old_value,
1446                        old_formula,
1447                    );
1448                }
1449            })?;
1450            self.engine.record_structural_change(StructuralScope::Cell {
1451                sheet: addr.sheet_id,
1452                row: addr.coord.row(),
1453                col: addr.coord.col(),
1454            });
1455
1456            if let Some(undo_ptr) = self.arrow_undo {
1457                let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1458                let undo = unsafe { &mut *undo_ptr };
1459                self.engine
1460                    .record_spill_ops_into_arrow_undo(undo, new_events);
1461                let delta_new: Option<LiteralValue> = None;
1462                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1463                let row0 = row.saturating_sub(1);
1464                let col0 = col.saturating_sub(1);
1465                undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1466            }
1467            Ok(())
1468        } else {
1469            self.engine
1470                .set_cell_formula(sheet, row, col, ast)
1471                .map_err(crate::engine::EditorError::from)
1472        }
1473    }
1474
1475    #[inline]
1476    pub fn set_row_hidden(
1477        &mut self,
1478        sheet: &str,
1479        row_1based: u32,
1480        hidden: bool,
1481        source: RowVisibilitySource,
1482    ) -> Result<(), crate::engine::EditorError> {
1483        if self.capture.is_some() {
1484            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1485            let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1486            let old_hidden = self
1487                .engine
1488                .row_visibility
1489                .get(&sheet_id)
1490                .map(|state| state.is_row_hidden(row0, Some(source)))
1491                .unwrap_or(false);
1492            if old_hidden == hidden {
1493                return Ok(());
1494            }
1495
1496            let _ = self
1497                .engine
1498                .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1499
1500            let Some(capture_ptr) = self.capture else {
1501                return Err(crate::engine::EditorError::TransactionFailed {
1502                    reason: "action_with_logger: missing mutation capture".to_string(),
1503                });
1504            };
1505            unsafe { &mut *capture_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1506                sheet_id,
1507                row0,
1508                source,
1509                old_hidden,
1510                new_hidden: hidden,
1511            });
1512
1513            Ok(())
1514        } else {
1515            self.engine
1516                .set_row_hidden(sheet, row_1based, hidden, source)
1517        }
1518    }
1519
1520    #[inline]
1521    pub fn set_rows_hidden(
1522        &mut self,
1523        sheet: &str,
1524        start_row_1based: u32,
1525        end_row_1based: u32,
1526        hidden: bool,
1527        source: RowVisibilitySource,
1528    ) -> Result<(), crate::engine::EditorError> {
1529        if self.capture.is_some() {
1530            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1531            let (start_row0, end_row0) =
1532                Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1533
1534            let Some(capture_ptr) = self.capture else {
1535                return Err(crate::engine::EditorError::TransactionFailed {
1536                    reason: "action_with_logger: missing mutation capture".to_string(),
1537                });
1538            };
1539            let capture = unsafe { &mut *capture_ptr };
1540
1541            for row0 in start_row0..=end_row0 {
1542                let old_hidden = self
1543                    .engine
1544                    .row_visibility
1545                    .get(&sheet_id)
1546                    .map(|state| state.is_row_hidden(row0, Some(source)))
1547                    .unwrap_or(false);
1548                if old_hidden == hidden {
1549                    continue;
1550                }
1551
1552                let _ = self
1553                    .engine
1554                    .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1555
1556                capture.record(crate::engine::ChangeEvent::SetRowVisibility {
1557                    sheet_id,
1558                    row0,
1559                    source,
1560                    old_hidden,
1561                    new_hidden: hidden,
1562                });
1563            }
1564
1565            Ok(())
1566        } else {
1567            self.engine
1568                .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1569        }
1570    }
1571
1572    #[inline]
1573    pub fn insert_rows(
1574        &mut self,
1575        sheet: &str,
1576        before: u32,
1577        count: u32,
1578    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1579        if count == 0 {
1580            return Ok(crate::engine::ShiftSummary::default());
1581        }
1582        if self.capture.is_some() {
1583            let Some(capture_ptr) = self.capture else {
1584                return Err(crate::engine::EditorError::TransactionFailed {
1585                    reason: "action_atomic: missing mutation capture".to_string(),
1586                });
1587            };
1588
1589            let before0 = before.saturating_sub(1);
1590            self.engine.check_insert_room(sheet, before0, count, true)?;
1591            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1592            let occupancy = self.engine.structural_row_occupancy(sheet, sheet_id);
1593            let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1594
1595            // Graph structural insert (logged) - no snapshot bump.
1596            let summary = {
1597                let capture = unsafe { &mut *capture_ptr };
1598                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1599                    Ok(crate::engine::ShiftSummary::default());
1600                self.engine.edit_with_capture(capture, |editor| {
1601                    editor.set_structural_occupancy(occupancy);
1602                    out = editor.insert_rows(sheet_id, before0, count);
1603                })?;
1604                out?
1605            };
1606
1607            // Arrow insert (truth) + undo op.
1608            self.engine.ensure_arrow_sheet(sheet);
1609            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1610                asheet.insert_rows(before0 as usize, count as usize);
1611            }
1612            self.engine
1613                .purge_derived_formats_after_row(sheet_id, before0);
1614            self.engine
1615                .shift_row_visibility_insert(sheet_id, before0, count);
1616            self.engine.mark_moved_formula_vertices_dirty(&summary);
1617            self.engine
1618                .clear_computed_overlay_after_row(sheet, before0 as usize);
1619            self.engine
1620                .record_structural_change(StructuralScope::Region(affected_region));
1621            if let Some(undo_ptr) = self.arrow_undo {
1622                unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1623            }
1624            Ok(summary)
1625        } else {
1626            self.engine.insert_rows(sheet, before, count)
1627        }
1628    }
1629
1630    #[inline]
1631    pub fn delete_rows(
1632        &mut self,
1633        sheet: &str,
1634        start: u32,
1635        count: u32,
1636    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1637        if count == 0 {
1638            return Ok(crate::engine::ShiftSummary::default());
1639        }
1640        if self.atomic_policy {
1641            return Err(crate::engine::EditorError::TransactionUnsupported {
1642                reason:
1643                    "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1644                        .to_string(),
1645            });
1646        }
1647        self.engine.delete_rows(sheet, start, count)
1648    }
1649
1650    #[inline]
1651    pub fn insert_columns(
1652        &mut self,
1653        sheet: &str,
1654        before: u32,
1655        count: u32,
1656    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1657        if count == 0 {
1658            return Ok(crate::engine::ShiftSummary::default());
1659        }
1660        if self.capture.is_some() {
1661            let Some(capture_ptr) = self.capture else {
1662                return Err(crate::engine::EditorError::TransactionFailed {
1663                    reason: "action_atomic: missing mutation capture".to_string(),
1664                });
1665            };
1666
1667            let before0 = before.saturating_sub(1);
1668            self.engine
1669                .check_insert_room(sheet, before0, count, false)?;
1670            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1671            let occupancy = self.engine.structural_column_occupancy();
1672            let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1673
1674            let summary = {
1675                let capture = unsafe { &mut *capture_ptr };
1676                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1677                    Ok(crate::engine::ShiftSummary::default());
1678                self.engine.edit_with_capture(capture, |editor| {
1679                    editor.set_structural_occupancy(occupancy);
1680                    out = editor.insert_columns(sheet_id, before0, count);
1681                })?;
1682                out?
1683            };
1684
1685            self.engine.ensure_arrow_sheet(sheet);
1686            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1687                asheet.insert_columns(before0 as usize, count as usize);
1688            }
1689            self.engine
1690                .purge_derived_formats_after_col(sheet_id, before0);
1691            self.engine.mark_moved_formula_vertices_dirty(&summary);
1692            self.engine
1693                .clear_computed_overlay_after_col(sheet, before0 as usize);
1694            self.engine
1695                .record_structural_change(StructuralScope::Region(affected_region));
1696            if let Some(undo_ptr) = self.arrow_undo {
1697                unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1698            }
1699            Ok(summary)
1700        } else {
1701            self.engine.insert_columns(sheet, before, count)
1702        }
1703    }
1704
1705    #[inline]
1706    pub fn delete_columns(
1707        &mut self,
1708        sheet: &str,
1709        start: u32,
1710        count: u32,
1711    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1712        if count == 0 {
1713            return Ok(crate::engine::ShiftSummary::default());
1714        }
1715        if self.atomic_policy {
1716            return Err(crate::engine::EditorError::TransactionUnsupported {
1717                reason:
1718                    "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1719                        .to_string(),
1720            });
1721        }
1722        self.engine.delete_columns(sheet, start, count)
1723    }
1724
1725    /// Start an action from within an action.
1726    ///
1727    /// Nested actions are currently disallowed (ticket 614), so this will return a
1728    /// `EditorError::TransactionFailed` while an outer action is active.
1729    #[inline]
1730    pub fn action<T>(
1731        &mut self,
1732        name: impl AsRef<str>,
1733        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1734    ) -> Result<T, crate::engine::EditorError> {
1735        self.engine.action(name, f)
1736    }
1737}
1738
1739struct ActionDepthGuard<'a, R> {
1740    engine: *mut Engine<R>,
1741    _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1742}
1743
1744impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1745    fn drop(&mut self) {
1746        // Safety: the guard is created from a unique `&mut Engine` borrow and lives no longer
1747        // than the surrounding `Engine::action` call.
1748        unsafe {
1749            let e = &mut *self.engine;
1750            e.action_depth = e.action_depth.saturating_sub(1);
1751        }
1752    }
1753}
1754
1755#[derive(Default)]
1756struct SourceCache {
1757    scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1758    tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1759    /// Materialized views of external range sources: one build per source
1760    /// version, shared by every formula that reads it.
1761    external_views: FxHashMap<(String, Option<u64>), RangeView<'static>>,
1762}
1763
1764#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1765struct VisibilityMaskCacheKey {
1766    sheet_id: SheetId,
1767    start_row0: u32,
1768    end_row0: u32,
1769    mode: VisibilityMaskMode,
1770    version: u64,
1771}
1772
1773#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1774enum StructuralScope {
1775    Cell { sheet: SheetId, row: u32, col: u32 },
1776    Region(Region),
1777    Sheet(SheetId),
1778    RemovedSheet(SheetId),
1779    OpaqueGlobal,
1780    AllSheets,
1781}
1782
1783/// Anchors blocked by a formula or another spill, indexed by the blocking
1784/// cell so that a single-cell change finds its waiters in O(1).
1785#[derive(Default)]
1786struct SpillBlockerWaiters {
1787    by_cell: FxHashMap<CellRef, Vec<VertexId>>,
1788    by_anchor: FxHashMap<VertexId, CellRef>,
1789}
1790
1791impl SpillBlockerWaiters {
1792    /// Largest region scanned cell by cell; a larger one scans the waiters.
1793    const REGION_PROBE_CELLS: u64 = 4096;
1794
1795    fn is_empty(&self) -> bool {
1796        self.by_anchor.is_empty()
1797    }
1798
1799    fn wait(&mut self, anchor: VertexId, blocker: CellRef) {
1800        match self.by_anchor.insert(anchor, blocker) {
1801            Some(old) if old == blocker => return,
1802            Some(old) => self.unlink(anchor, old),
1803            None => {}
1804        }
1805        self.by_cell.entry(blocker).or_default().push(anchor);
1806    }
1807
1808    fn forget(&mut self, anchor: VertexId) {
1809        if self.by_anchor.is_empty() {
1810            return;
1811        }
1812        if let Some(old) = self.by_anchor.remove(&anchor) {
1813            self.unlink(anchor, old);
1814        }
1815    }
1816
1817    fn unlink(&mut self, anchor: VertexId, cell: CellRef) {
1818        if let Some(waiters) = self.by_cell.get_mut(&cell) {
1819            waiters.retain(|&v| v != anchor);
1820            if waiters.is_empty() {
1821                self.by_cell.remove(&cell);
1822            }
1823        }
1824    }
1825
1826    /// Remove and return the anchors waiting on a cell inside `scope`.
1827    fn take_affected(&mut self, scope: StructuralScope) -> Vec<VertexId> {
1828        let cells: Vec<CellRef> = match scope {
1829            StructuralScope::Cell { sheet, row, col } => {
1830                match Coord::try_new(row, col, true, true) {
1831                    Ok(coord) if self.by_cell.contains_key(&CellRef::new(sheet, coord)) => {
1832                        vec![CellRef::new(sheet, coord)]
1833                    }
1834                    _ => Vec::new(),
1835                }
1836            }
1837            StructuralScope::Region(region) => {
1838                let (rows, cols) = region.axis_ranges();
1839                let (r0, r1) = rows.query_bounds();
1840                let (c0, c1) = cols.query_bounds();
1841                // Widen before `+ 1`: an unbounded axis (every structural edit
1842                // records `rows_from` / `cols_from`) spans 0..=u32::MAX, and
1843                // the u32 sum wrapped to an area of 0.
1844                let area = (u64::from(r1.saturating_sub(r0)) + 1)
1845                    .saturating_mul(u64::from(c1.saturating_sub(c0)) + 1);
1846                if area <= Self::REGION_PROBE_CELLS {
1847                    let sheet = region.sheet_id();
1848                    // A small region can still lie past the grid; no waiter
1849                    // is registered there, so those points are skipped.
1850                    (r0..=r1)
1851                        .flat_map(|r| (c0..=c1).map(move |c| (r, c)))
1852                        .filter_map(|(r, c)| Coord::try_new(r, c, true, true).ok())
1853                        .map(|coord| CellRef::new(sheet, coord))
1854                        .filter(|cell| self.by_cell.contains_key(cell))
1855                        .collect()
1856                } else {
1857                    self.by_cell
1858                        .keys()
1859                        .filter(|cell| {
1860                            region.intersects(&Region::point(
1861                                cell.sheet_id,
1862                                cell.coord.row(),
1863                                cell.coord.col(),
1864                            ))
1865                        })
1866                        .copied()
1867                        .collect()
1868                }
1869            }
1870            StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => self
1871                .by_cell
1872                .keys()
1873                .filter(|cell| cell.sheet_id == sheet)
1874                .copied()
1875                .collect(),
1876            StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => {
1877                self.by_cell.keys().copied().collect()
1878            }
1879        };
1880        let mut woken = Vec::new();
1881        for cell in cells {
1882            if let Some(waiters) = self.by_cell.remove(&cell) {
1883                for anchor in waiters {
1884                    self.by_anchor.remove(&anchor);
1885                    woken.push(anchor);
1886                }
1887            }
1888        }
1889        woken
1890    }
1891}
1892
1893#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
1894enum LoggedEditImpact {
1895    NoOp,
1896    DataOnly,
1897    Topology,
1898}
1899
1900#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1901enum LoggedEditDirection {
1902    Original,
1903    InverseReplay,
1904    ForwardReplay,
1905}
1906
1907#[derive(Clone, Copy)]
1908struct InvalidationBaseline {
1909    snapshot_id: u64,
1910    topology_epoch: u64,
1911}
1912
1913struct SourceCacheSession {
1914    cache: Arc<std::sync::RwLock<SourceCache>>,
1915}
1916
1917impl Drop for SourceCacheSession {
1918    fn drop(&mut self) {
1919        if let Ok(mut g) = self.cache.write() {
1920            *g = SourceCache::default();
1921        }
1922    }
1923}
1924
1925#[derive(Debug)]
1926#[non_exhaustive]
1927pub struct EvalResult {
1928    pub computed_vertices: usize,
1929    pub cycle_errors: usize,
1930    pub elapsed: std::time::Duration,
1931}
1932
1933#[derive(Clone, Debug, PartialEq, Eq)]
1934#[non_exhaustive]
1935pub struct TableMetadata {
1936    pub name: String,
1937    pub sheet: String,
1938    pub start_row: u32,
1939    pub start_col: u32,
1940    pub end_row: u32,
1941    pub end_col: u32,
1942    pub header_row: bool,
1943    pub headers: Vec<String>,
1944    pub totals_row: bool,
1945}
1946
1947/// Read-only engine counters used by benchmark/instrumentation tooling.
1948///
1949/// These counters are deliberately observational: collecting them must not mutate engine state or
1950/// alter formula evaluation semantics.
1951///
1952/// The `formula_plane_*` counters are always `0`: FormulaPlane spans were removed and the
1953/// dependency authority is the only runtime path. They are kept for source compatibility.
1954#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1955#[non_exhaustive]
1956pub struct EngineBaselineStats {
1957    pub graph_vertex_count: usize,
1958    pub graph_formula_vertex_count: usize,
1959    pub graph_edge_count: usize,
1960    pub dirty_vertex_count: usize,
1961    pub evaluation_vertex_count: usize,
1962    pub formula_ast_root_count: usize,
1963    pub formula_ast_node_count: usize,
1964    pub staged_formula_count: usize,
1965    pub formula_plane_active_span_count: usize,
1966    pub formula_plane_producer_result_entries: usize,
1967    pub formula_plane_consumer_read_entries: usize,
1968    pub formula_plane_mixed_topology_cache_builds: u64,
1969    pub formula_plane_mixed_topology_cache_hits: u64,
1970    pub formula_plane_mixed_topology_cache_overflows: u64,
1971    pub formula_plane_dirty_pending_events: usize,
1972    pub formula_plane_dirty_region_events_recorded: u64,
1973    pub formula_plane_dirty_span_region_events_recorded: u64,
1974    pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1975    pub formula_plane_dirty_global_invalidations: u64,
1976    pub formula_plane_structural_span_candidates: u64,
1977    pub formula_plane_cycle_member_span_demotions: u64,
1978    pub formula_plane_array_result_span_demotions: u64,
1979    /// Members of exactly converged iterative SCCs currently retained across
1980    /// recalcs (#368).
1981    pub retained_scc_members: usize,
1982}
1983
1984#[derive(Debug, Clone, Default)]
1985#[non_exhaustive]
1986pub struct VirtualDepTelemetry {
1987    pub candidate_vertices_total: usize,
1988    pub vdeps_vertices_total: usize,
1989    pub vdeps_edges_total: usize,
1990    pub builder_elapsed_ms_total: u128,
1991    pub schedule_virtual_passes: usize,
1992    pub schedule_static_passes: usize,
1993    pub schedule_cache_hits: usize,
1994    pub schedule_cache_misses: usize,
1995    pub reused_schedule_vertices_total: usize,
1996    pub replan_iterations: usize,
1997    pub changed_vdeps_total: usize,
1998    pub bailout_reason: Option<&'static str>,
1999    pub fallback_mode_activations: u64,
2000}
2001
2002/// Per-recalc telemetry for SCC evaluation under `CycleDetection::Runtime`
2003/// (spec `formualizer-cycle-semantics-spec.md` §10).
2004///
2005/// Collection is unconditional: SCC tasks are rare relative to ordinary
2006/// vertex evaluation and the counters are a handful of integer adds per
2007/// task, so no config flag gates them (unlike [`VirtualDepTelemetry`],
2008/// which pays per-schedule costs). Counters reset at the start of every
2009/// evaluation request.
2010#[derive(Debug, Clone, Default, PartialEq)]
2011#[non_exhaustive]
2012pub struct CycleTelemetry {
2013    /// SCC tasks executed (static SCCs that reached Runtime evaluation).
2014    pub static_sccs: usize,
2015    /// SCC tasks whose live subgraph was acyclic — values produced.
2016    pub phantom_sccs: usize,
2017    /// Distinct live cycles witnessed across all SCC tasks.
2018    pub live_cycles_witnessed: usize,
2019    /// Cells stamped `#CIRC!` by Runtime SCC tasks.
2020    pub circ_cells_stamped: usize,
2021    /// Evaluation sweeps over (subsets of) SCC members, totalled across tasks
2022    /// (pass 1 included).
2023    pub settle_passes_total: usize,
2024    /// Largest pass count any single SCC task needed.
2025    pub max_passes_single_scc: usize,
2026    /// SCC tasks that entered iterative calculation (`CyclePolicy::Iterate`
2027    /// with a witnessed live cycle). RFC #113, Stage 3.
2028    pub iterated_sccs: usize,
2029    /// Iterating SCC tasks that stopped because every member passed the
2030    /// spec-§6 convergence test.
2031    pub converged_sccs: usize,
2032    /// SCC tasks that stopped at a pass cap. Under `CyclePolicy::Iterate`
2033    /// this is the Excel `max_iterations` cap (NOT an error — last values
2034    /// are kept; includes the no-convergence-test `max_iterations: 1`
2035    /// contract). Under `CyclePolicy::Error` it is the defensive acyclic
2036    /// settle cap (|SCC| + 2), which only a bug can hit.
2037    pub capped_sccs: usize,
2038    /// Largest `|Δ|` observed in any member's final-pass convergence
2039    /// comparison across iterating SCC tasks (numeric-class members only).
2040    /// `0.0` when no comparison ran (e.g. `max_iterations: 1`).
2041    pub max_abs_delta_at_stop: f64,
2042    /// Identical-bit NaN vs NaN member comparisons that were treated as
2043    /// converged (spec §6 NaN rule).
2044    pub nan_converged: usize,
2045    /// Retained iterative SCCs (#368) that had no dirty member at request begin and were therefore not
2046    /// re-run: their last exact fixed point is served as-is. Counted at
2047    /// request begin, so a demand-driven request that never reaches a
2048    /// retained SCC still reports it as reused.
2049    pub reused_sccs: usize,
2050    /// Members of the SCCs counted in `reused_sccs`.
2051    pub reused_scc_members: usize,
2052    /// Total wall-clock time spent inside Runtime SCC tasks.
2053    pub elapsed_ms: u128,
2054}
2055
2056#[derive(Debug, Clone, Copy)]
2057struct ScheduleBuildMeta {
2058    candidate_vertices: usize,
2059    vdeps_vertices: usize,
2060    vdeps_edges: usize,
2061    builder_elapsed_ms: u128,
2062    used_virtual_schedule: bool,
2063    schedule_cache_hit: bool,
2064    schedule_cache_eligible: bool,
2065}
2066
2067#[cfg(any(test, feature = "benchmark_internal"))]
2068#[doc(hidden)]
2069#[derive(Debug, Clone, Default)]
2070pub struct RecalcReuseProbe {
2071    pub schedule_requests: usize,
2072    pub schedule_cache_hits: usize,
2073    pub schedule_cache_misses: usize,
2074    pub schedule_cache_ineligible: usize,
2075    pub schedule_builds: usize,
2076    /// Program 3 plan reuse: misses served by restricting the base schedule.
2077    pub schedule_base_restrictions: usize,
2078    pub schedule_shared_handles: usize,
2079    pub schedule_retained_bytes: usize,
2080    pub legacy_target_requests: usize,
2081    pub target_schedule_builds: usize,
2082    pub demand_builds: usize,
2083    pub demand_vertices: usize,
2084    pub demand_clean_formulas: usize,
2085    pub demand_explicit_edges: usize,
2086    pub demand_virtual_builder_calls: usize,
2087}
2088
2089#[cfg(any(test, feature = "benchmark_internal"))]
2090fn schedule_probe_retained_bytes(schedule: &crate::engine::Schedule) -> usize {
2091    fn vector_bytes<T>(values: &Vec<T>) -> usize {
2092        values.capacity() * std::mem::size_of::<T>()
2093    }
2094
2095    [
2096        vector_bytes(&schedule.units),
2097        vector_bytes(&schedule.layers),
2098        vector_bytes(&schedule.cycles),
2099    ]
2100    .into_iter()
2101    .chain(
2102        schedule
2103            .layers
2104            .iter()
2105            .map(|layer| vector_bytes(&layer.vertices)),
2106    )
2107    .chain(schedule.cycles.iter().map(vector_bytes))
2108    .sum()
2109}
2110
2111#[derive(Debug, Clone)]
2112struct CachedScheduleEntry {
2113    topology_epoch: u64,
2114    /// Authority `(store revision, rev.dyn)` the schedule was planned from
2115    /// (design §8.4; always 0 without `unified_authority`).
2116    authority_revision: (u64, u64),
2117    /// The request's vertex list as runs of consecutive ids (formula ids
2118    /// come in column runs, so a whole-workbook request is a few runs).
2119    candidate_vertices: VertexIdRuns,
2120    schedule: Arc<crate::engine::scheduler::Schedule>,
2121}
2122
2123/// A vertex list stored as `(first id, run length)` runs of consecutive
2124/// ids, in list order.
2125#[derive(Debug, Clone, Default)]
2126struct VertexIdRuns(Vec<(u32, u32)>);
2127
2128impl VertexIdRuns {
2129    fn from_slice(ids: &[VertexId]) -> Self {
2130        let mut runs: Vec<(u32, u32)> = Vec::new();
2131        for v in ids {
2132            match runs.last_mut() {
2133                Some((first, len)) if first.checked_add(*len) == Some(v.0) => *len += 1,
2134                _ => runs.push((v.0, 1)),
2135            }
2136        }
2137        runs.shrink_to_fit();
2138        Self(runs)
2139    }
2140
2141    fn equals(&self, ids: &[VertexId]) -> bool {
2142        let mut rest = ids;
2143        for &(first, len) in &self.0 {
2144            let len = len as usize;
2145            if rest.len() < len {
2146                return false;
2147            }
2148            let (head, tail) = rest.split_at(len);
2149            if head
2150                .iter()
2151                .enumerate()
2152                .any(|(i, v)| v.0 != first.wrapping_add(i as u32))
2153            {
2154                return false;
2155            }
2156            rest = tail;
2157        }
2158        rest.is_empty()
2159    }
2160
2161    fn heap_bytes(&self) -> usize {
2162        self.0.capacity() * std::mem::size_of::<(u32, u32)>()
2163    }
2164
2165    /// The number of ids.
2166    fn len(&self) -> usize {
2167        self.0.iter().map(|&(_, len)| len as usize).sum()
2168    }
2169}
2170
2171#[cfg(test)]
2172mod vertex_id_runs_tests {
2173    use super::{VertexId, VertexIdRuns};
2174
2175    #[test]
2176    fn runs_compare_like_the_list() {
2177        let ids = |v: &[u32]| v.iter().map(|&i| VertexId(i)).collect::<Vec<_>>();
2178        let list = ids(&[5, 6, 7, 2, 3, 9, 10, 10]);
2179        let runs = VertexIdRuns::from_slice(&list);
2180        assert_eq!(runs.0, vec![(5, 3), (2, 2), (9, 2), (10, 1)]);
2181        assert!(runs.equals(&list));
2182        assert!(!runs.equals(&list[..7]));
2183        assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 11])));
2184        assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 10, 11])));
2185        assert!(VertexIdRuns::from_slice(&[]).equals(&[]));
2186        assert!(!VertexIdRuns::from_slice(&[]).equals(&list));
2187        let edge = ids(&[u32::MAX - 1, u32::MAX, 0]);
2188        assert!(VertexIdRuns::from_slice(&edge).equals(&edge));
2189    }
2190}
2191
2192/// Uncacheable requests keep their schedule inline without a shared allocation.
2193enum EvaluationSchedule {
2194    Owned(crate::engine::scheduler::Schedule),
2195    Shared(Arc<crate::engine::scheduler::Schedule>),
2196}
2197
2198impl std::ops::Deref for EvaluationSchedule {
2199    type Target = crate::engine::scheduler::Schedule;
2200
2201    fn deref(&self) -> &Self::Target {
2202        match self {
2203            Self::Owned(schedule) => schedule,
2204            Self::Shared(schedule) => schedule,
2205        }
2206    }
2207}
2208
2209type ScheduleBuildOutput = (
2210    crate::engine::scheduler::Schedule,
2211    FxHashMap<VertexId, Vec<VertexId>>,
2212    ScheduleBuildMeta,
2213);
2214
2215type EvaluationScheduleBuildOutput = (
2216    EvaluationSchedule,
2217    FxHashMap<VertexId, Vec<VertexId>>,
2218    ScheduleBuildMeta,
2219);
2220
2221/// Opaque, revision-bound recalculation recipe.
2222#[derive(Debug)]
2223pub struct RecalcPlan {
2224    key: RecalcPlanKey,
2225    kind: RecalcPlanKind,
2226}
2227
2228#[derive(Debug)]
2229struct RecalcPlanKey {
2230    engine_token: Arc<()>,
2231    revisions: PlanningRevisionSnapshot,
2232}
2233
2234#[derive(Clone, Debug, PartialEq, Eq)]
2235struct PlanningRevisionSnapshot {
2236    engine_topology_epoch: u64,
2237    graph_topology_revision: u64,
2238    staged: u64,
2239    symbols: u64,
2240    semantic: u64,
2241    provider: Option<u64>,
2242    deterministic_mode: crate::engine::DeterministicMode,
2243    budgets: crate::engine::EvaluationBudgets,
2244}
2245
2246#[derive(Debug)]
2247enum RecalcPlanKind {
2248    CompatibilityFull {
2249        schedule: crate::engine::Schedule,
2250        has_dynamic_refs: bool,
2251    },
2252    Target {
2253        targets: Vec<crate::engine::EvaluationTarget>,
2254        scope: crate::engine::PrepareScope,
2255        topology: RecalcTopology,
2256        dynamic_policy: DynamicPlanPolicy,
2257    },
2258}
2259
2260#[derive(Debug)]
2261enum RecalcTopology {
2262    RunLocalRecipe,
2263    Workbook,
2264}
2265
2266#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2267enum DynamicPlanPolicy {
2268    BoundedTargetReplan,
2269}
2270
2271impl RecalcPlan {
2272    /// Returns the retained compatibility schedule depth. Target plans retain a
2273    /// run-local recipe rather than a schedule, so their layer count is zero.
2274    pub fn layer_count(&self) -> usize {
2275        match &self.kind {
2276            RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2277            RecalcPlanKind::Target { .. } => 0,
2278        }
2279    }
2280
2281    pub fn has_dynamic_refs(&self) -> bool {
2282        match &self.kind {
2283            RecalcPlanKind::CompatibilityFull {
2284                has_dynamic_refs, ..
2285            } => *has_dynamic_refs,
2286            RecalcPlanKind::Target { .. } => false,
2287        }
2288    }
2289
2290    #[cfg(test)]
2291    pub(crate) fn force_stale_reasons_for_test(
2292        &mut self,
2293        reasons: &[formualizer_common::PlanStaleReason],
2294    ) {
2295        use formualizer_common::PlanStaleReason;
2296        for reason in reasons {
2297            match reason {
2298                PlanStaleReason::Engine => {
2299                    self.key.engine_token = Arc::new(());
2300                }
2301                PlanStaleReason::Provider => {
2302                    self.key.revisions.provider = Some(
2303                        self.key
2304                            .revisions
2305                            .provider
2306                            .unwrap_or_default()
2307                            .wrapping_add(1),
2308                    );
2309                }
2310                PlanStaleReason::Semantic => {
2311                    self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2312                }
2313                PlanStaleReason::Budget => {
2314                    let current = self.key.revisions.budgets.work.max_work_units;
2315                    self.key.revisions.budgets.work.max_work_units =
2316                        Some(current.unwrap_or_default().wrapping_add(1));
2317                }
2318                PlanStaleReason::Staged => {
2319                    self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2320                }
2321                PlanStaleReason::Symbols => {
2322                    self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2323                }
2324                PlanStaleReason::Graph => {
2325                    self.key.revisions.graph_topology_revision =
2326                        self.key.revisions.graph_topology_revision.wrapping_add(1);
2327                }
2328                _ => {}
2329            }
2330        }
2331    }
2332}
2333
2334#[cfg(any(test, feature = "test-support"))]
2335pub(crate) mod criteria_mask_test_hooks {
2336    use std::cell::Cell;
2337
2338    thread_local! {
2339        static MASK_CALLS_ROWS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
2340        static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2341        static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2342    }
2343
2344    pub(crate) fn take_mask_work() -> (usize, usize) {
2345        MASK_CALLS_ROWS.with(|c| c.replace((0, 0)))
2346    }
2347
2348    pub(crate) fn note_mask(rows: usize) {
2349        MASK_CALLS_ROWS.with(|c| {
2350            let (calls, work) = c.get();
2351            c.set((calls + 1, work + rows));
2352        });
2353    }
2354
2355    pub fn reset_text_segment_counters() {
2356        TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2357        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2358    }
2359
2360    pub fn text_segment_counters() -> (usize, usize) {
2361        let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2362        let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2363        (a, b)
2364    }
2365
2366    pub(crate) fn inc_total() {
2367        TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2368    }
2369    pub(crate) fn inc_all_null() {
2370        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2371    }
2372}
2373
2374#[cfg(test)]
2375pub(crate) mod visibility_mask_test_hooks {
2376    use std::cell::Cell;
2377
2378    thread_local! {
2379        static HITS: Cell<usize> = const { Cell::new(0) };
2380        static MISSES: Cell<usize> = const { Cell::new(0) };
2381        static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2382    }
2383
2384    pub fn reset() {
2385        HITS.with(|c| c.set(0));
2386        MISSES.with(|c| c.set(0));
2387        EVICTIONS.with(|c| c.set(0));
2388    }
2389
2390    pub fn counters() -> (usize, usize, usize) {
2391        let hits = HITS.with(|c| c.get());
2392        let misses = MISSES.with(|c| c.get());
2393        let evictions = EVICTIONS.with(|c| c.get());
2394        (hits, misses, evictions)
2395    }
2396
2397    pub(crate) fn inc_hit() {
2398        HITS.with(|c| c.set(c.get() + 1));
2399    }
2400
2401    pub(crate) fn inc_miss() {
2402        MISSES.with(|c| c.set(c.get() + 1));
2403    }
2404
2405    pub(crate) fn inc_eviction() {
2406        EVICTIONS.with(|c| c.set(c.get() + 1));
2407    }
2408}
2409
2410fn is_numeric_text_equality(pred: &crate::args::CriteriaPredicate) -> bool {
2411    match pred {
2412        crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(text)) => {
2413            text.trim().parse::<f64>().is_ok_and(f64::is_finite)
2414        }
2415        _ => false,
2416    }
2417}
2418
2419/// Resolve the null (non-number) rows of an `=n` / `<>n` criteria mask over
2420/// one row segment: numeric text compares by value, every other non-number
2421/// cell (blank, boolean, error) is unequal. Segments without text stay
2422/// vectorized.
2423fn fill_numeric_equality_nulls(
2424    view: &RangeView<'_>,
2425    row_start: usize,
2426    row_len: usize,
2427    col_in_view: usize,
2428    mask: &arrow_array::BooleanArray,
2429    n: f64,
2430    ne: bool,
2431) -> Option<arrow_array::BooleanArray> {
2432    use arrow_array::Array as _;
2433    let texts = view.slice_lowered_text(row_start, row_len);
2434    let text = texts.get(col_in_view).and_then(|t| t.as_ref());
2435    match text {
2436        None => {
2437            if !ne {
2438                // Nulls are already "no match".
2439                return Some(mask.clone());
2440            }
2441            let nulls = arrow::compute::is_null(mask).ok()?;
2442            crate::compute_prelude::boolean::or_kleene(mask, &nulls).ok()
2443        }
2444        Some(text) => {
2445            if text.len() != mask.len() {
2446                return None;
2447            }
2448            let mut out = arrow_array::builder::BooleanBuilder::with_capacity(mask.len());
2449            for i in 0..mask.len() {
2450                if mask.is_valid(i) {
2451                    out.append_value(mask.value(i));
2452                } else if text.is_valid(i) {
2453                    let equal = crate::locale::Locale::invariant()
2454                        .parse_number_invariant(text.value(i))
2455                        .is_some_and(|x| (x - n).abs() < 1e-12);
2456                    out.append_value(equal != ne);
2457                } else {
2458                    out.append_value(ne);
2459                }
2460            }
2461            Some(out.finish())
2462        }
2463    }
2464}
2465
2466fn compute_criteria_mask(
2467    view: &RangeView<'_>,
2468    col_in_view: usize,
2469    pred: &crate::args::CriteriaPredicate,
2470) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2471    use crate::compute_prelude::{boolean, cmp, concat_arrays};
2472    use arrow::compute::kernels::comparison::{ilike, nilike};
2473    use arrow_array::{
2474        Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2475    };
2476
2477    // Helper: apply a numeric predicate to a single Float64Array chunk
2478    fn apply_numeric_pred(
2479        chunk: &Float64Array,
2480        pred: &crate::args::CriteriaPredicate,
2481    ) -> Option<BooleanArray> {
2482        match pred {
2483            crate::args::CriteriaPredicate::Gt(n) => {
2484                cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2485            }
2486            crate::args::CriteriaPredicate::Ge(n) => {
2487                cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2488            }
2489            crate::args::CriteriaPredicate::Lt(n) => {
2490                cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2491            }
2492            crate::args::CriteriaPredicate::Le(n) => {
2493                cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2494            }
2495            crate::args::CriteriaPredicate::Eq(v) => match v {
2496                formualizer_common::LiteralValue::Number(x) => {
2497                    cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2498                }
2499                formualizer_common::LiteralValue::Int(i) => {
2500                    cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2501                }
2502                _ => None,
2503            },
2504            crate::args::CriteriaPredicate::Ne(v) => match v {
2505                formualizer_common::LiteralValue::Number(x) => {
2506                    cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2507                }
2508                formualizer_common::LiteralValue::Int(i) => {
2509                    cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2510                }
2511                _ => None,
2512            },
2513            _ => None,
2514        }
2515    }
2516
2517    // Check if this is a numeric predicate that can be applied per-chunk
2518    let is_numeric_pred = matches!(
2519        pred,
2520        crate::args::CriteriaPredicate::Gt(_)
2521            | crate::args::CriteriaPredicate::Ge(_)
2522            | crate::args::CriteriaPredicate::Lt(_)
2523            | crate::args::CriteriaPredicate::Le(_)
2524            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2525            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2526            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2527            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2528    );
2529
2530    // OPTIMIZED PATH: For numeric predicates, apply per-chunk and concatenate boolean masks.
2531    // This avoids materializing the full numeric column (64-bit per element) and instead
2532    // concatenates boolean masks (1-bit per element) - a 64x memory reduction.
2533    if is_numeric_pred {
2534        let mut bool_parts: Vec<BooleanArray> = Vec::new();
2535        // `=n` / `<>n`: the number lane is null for blank, text, boolean and
2536        // error cells. Blanks, booleans and errors never equal a number;
2537        // numeric text does (`5` matches "5"), as in `criteria_match`.
2538        let number = |v: &formualizer_common::LiteralValue| match v {
2539            formualizer_common::LiteralValue::Number(n) => Some(*n),
2540            formualizer_common::LiteralValue::Int(i) => Some(*i as f64),
2541            _ => None,
2542        };
2543        let equality = match pred {
2544            crate::args::CriteriaPredicate::Eq(v) => number(v).map(|n| (n, false)),
2545            crate::args::CriteriaPredicate::Ne(v) => number(v).map(|n| (n, true)),
2546            _ => None,
2547        };
2548        for res in view.numbers_slices() {
2549            let (rs, rl, cols_seg) = res.ok()?;
2550            if col_in_view < cols_seg.len() {
2551                let chunk = cols_seg[col_in_view].as_ref();
2552                let mut mask = apply_numeric_pred(chunk, pred)?;
2553                if let Some((n, ne)) = equality
2554                    && mask.null_count() > 0
2555                {
2556                    mask = fill_numeric_equality_nulls(view, rs, rl, col_in_view, &mask, n, ne)?;
2557                }
2558                bool_parts.push(mask);
2559            }
2560        }
2561
2562        if bool_parts.is_empty() {
2563            return None;
2564        } else if bool_parts.len() == 1 {
2565            return Some(std::sync::Arc::new(bool_parts.remove(0)));
2566        } else {
2567            // Concatenate boolean masks (much cheaper than concatenating Float64 arrays)
2568            let anys: Vec<&dyn arrow_array::Array> = bool_parts
2569                .iter()
2570                .map(|a| a as &dyn arrow_array::Array)
2571                .collect();
2572            let conc: ArrayRef = concat_arrays(&anys).ok()?;
2573            let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2574            return Some(std::sync::Arc::new(ba));
2575        }
2576    }
2577
2578    // Wildcards and numeric text equality can match non-text cells. The lowered
2579    // base lane is text-only, unlike the scalar matcher. Keep the vectorized
2580    // path for text-only data, but cache a scalar-equivalent mask for mixed data.
2581    if is_numeric_text_equality(pred)
2582        || matches!(pred, crate::args::CriteriaPredicate::TextLike { .. })
2583    {
2584        for tags in view.type_tags_slices() {
2585            let (_, _, cols) = tags.ok()?;
2586            let tags = cols.get(col_in_view)?;
2587            if tags.values().iter().any(|tag| {
2588                *tag == crate::arrow_store::TypeTag::Empty as u8
2589                    || *tag == crate::arrow_store::TypeTag::Number as u8
2590                    || *tag == crate::arrow_store::TypeTag::Boolean as u8
2591            }) {
2592                let mut mask = BooleanBuilder::new();
2593                for chunk in view.iter_row_chunks() {
2594                    let chunk = chunk.ok()?;
2595                    for row in chunk.row_start..chunk.row_start + chunk.row_len {
2596                        mask.append_value(crate::builtins::criteria_match(
2597                            pred,
2598                            &view.get_cell(row, col_in_view),
2599                        ));
2600                    }
2601                }
2602                return Some(std::sync::Arc::new(mask.finish()));
2603            }
2604        }
2605    }
2606
2607    // Keep spreadsheet tilde escapes in the shared scalar matcher.
2608    if matches!(pred, crate::args::CriteriaPredicate::TextLike { pattern, .. }
2609        if pattern.contains('~'))
2610    {
2611        return None;
2612    }
2613
2614    // TEXT PATH: build masks per row-chunk using lowered text slices.
2615    // This avoids concatenating full-string columns just to compute a boolean mask.
2616    // `"="` / `"<>"`: truly blank cells only (empty text is not blank).
2617    if matches!(
2618        pred,
2619        crate::args::CriteriaPredicate::IsBlank | crate::args::CriteriaPredicate::NotBlank
2620    ) {
2621        let want_blank = matches!(pred, crate::args::CriteriaPredicate::IsBlank);
2622        let mut bool_parts: Vec<BooleanArray> = Vec::new();
2623        for tags in view.type_tags_slices() {
2624            let (_, _, cols) = tags.ok()?;
2625            let tags = cols.get(col_in_view)?;
2626            let mut bb = BooleanBuilder::with_capacity(tags.len());
2627            for i in 0..tags.len() {
2628                let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8;
2629                bb.append_value(blank == want_blank);
2630            }
2631            bool_parts.push(bb.finish());
2632        }
2633        return match bool_parts.len() {
2634            0 => None,
2635            1 => Some(std::sync::Arc::new(bool_parts.remove(0))),
2636            _ => {
2637                let anys: Vec<&dyn arrow_array::Array> = bool_parts
2638                    .iter()
2639                    .map(|a| a as &dyn arrow_array::Array)
2640                    .collect();
2641                let conc: ArrayRef = concat_arrays(&anys).ok()?;
2642                let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2643                Some(std::sync::Arc::new(ba))
2644            }
2645        };
2646    }
2647
2648    // Arrow LIKE uses backslash escapes; SQL punctuation is literal in Excel.
2649    let like_pattern = |text: &str, wildcards: bool| {
2650        let mut pattern = String::with_capacity(text.len());
2651        for ch in text.chars() {
2652            match ch {
2653                '%' | '_' | '\\' => {
2654                    pattern.push('\\');
2655                    pattern.push(ch);
2656                }
2657                '*' if wildcards => pattern.push('%'),
2658                '?' if wildcards => pattern.push('_'),
2659                _ => pattern.push(ch),
2660            }
2661        }
2662        pattern
2663    };
2664    let (text_kind, text_pat, empty_special) = match pred {
2665        crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2666            (0u8, like_pattern(&t.to_lowercase(), false), t.is_empty())
2667        }
2668        crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2669            (1u8, like_pattern(&t.to_lowercase(), false), false)
2670        }
2671        crate::args::CriteriaPredicate::TextLike {
2672            pattern,
2673            case_insensitive,
2674        } => {
2675            let p = if *case_insensitive {
2676                pattern.to_lowercase()
2677            } else {
2678                pattern.clone()
2679            };
2680            (2u8, like_pattern(&p, true), false)
2681        }
2682        _ => return None,
2683    };
2684
2685    let text_pat_is_empty = text_pat.is_empty();
2686    let ne_matches_blank = text_kind == 1 && !text_pat_is_empty;
2687    let pat = StringArray::new_scalar(text_pat);
2688    let mut bool_parts: Vec<BooleanArray> = Vec::new();
2689
2690    let mut tag_slices = view.type_tags_slices();
2691    for res in view.iter_row_chunks() {
2692        let cs = res.ok()?;
2693        if cs.row_len == 0 {
2694            continue;
2695        }
2696        #[cfg(test)]
2697        criteria_mask_test_hooks::inc_total();
2698
2699        let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2700        if col_in_view >= slices.len() {
2701            return None;
2702        }
2703
2704        let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2705        if empty_special || (text_kind == 1 && text_pat_is_empty) {
2706            let (tag_start, tag_len, tags) = tag_slices.next()?.ok()?;
2707            if tag_start != cs.row_start || tag_len != cs.row_len {
2708                return None;
2709            }
2710            let tags = tags.get(col_in_view)?;
2711            // A null text lane is not a blank cell: base numeric/boolean/error
2712            // cells also have null text. Consult the overlay-aware type tags,
2713            // and inspect strings only to distinguish empty text from text.
2714            let strings = seg_opt.and_then(|a| a.as_any().downcast_ref::<StringArray>());
2715            let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2716            for i in 0..cs.row_len {
2717                let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8
2718                    || (tags.value(i) == crate::arrow_store::TypeTag::Text as u8
2719                        && strings.is_some_and(|s| s.is_valid(i) && s.value(i).is_empty()));
2720                bb.append_value(if text_kind == 0 { blank } else { !blank });
2721            }
2722            #[cfg(test)]
2723            if seg_opt.is_none() {
2724                criteria_mask_test_hooks::inc_all_null();
2725            }
2726            bool_parts.push(bb.finish());
2727            continue;
2728        }
2729        let seg = match seg_opt {
2730            Some(s) => s,
2731            None => {
2732                #[cfg(test)]
2733                criteria_mask_test_hooks::inc_all_null();
2734                if (text_kind == 0 && empty_special) || ne_matches_blank {
2735                    // Eq("") treats nulls (Empty) as equal.
2736                    let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2737                    bb.append_n(cs.row_len, true);
2738                    bool_parts.push(bb.finish());
2739                } else {
2740                    // For non-empty patterns, ilike/nilike return null on null inputs.
2741                    bool_parts.push(BooleanArray::new_null(cs.row_len));
2742                }
2743                continue;
2744            }
2745        };
2746
2747        let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2748        let mut m = match text_kind {
2749            0 => ilike(seg_sa, &pat).ok()?,
2750            1 => nilike(seg_sa, &pat).ok()?,
2751            2 => ilike(seg_sa, &pat).ok()?,
2752            _ => return None,
2753        };
2754
2755        // Only fold blank/Empty (null) cells into the mask when the segment
2756        // actually contains any. The null-fill loop + or_kleene are pure
2757        // overhead on blank-free chunks, so a `<>text` (or `=""`) aggregation
2758        // over a column with no blanks stays fully vectorized on the ilike/
2759        // nilike result.
2760        if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2761            // Treat nulls as equal to empty string
2762            let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2763            for i in 0..seg_sa.len() {
2764                bb.append_value(seg_sa.is_null(i));
2765            }
2766            let nulls = bb.finish();
2767            m = boolean::or_kleene(&m, &nulls).ok()?;
2768        }
2769
2770        bool_parts.push(m);
2771    }
2772
2773    if bool_parts.is_empty() {
2774        None
2775    } else if bool_parts.len() == 1 {
2776        Some(std::sync::Arc::new(bool_parts.remove(0)))
2777    } else {
2778        let anys: Vec<&dyn arrow_array::Array> = bool_parts
2779            .iter()
2780            .map(|a| a as &dyn arrow_array::Array)
2781            .collect();
2782        let conc: ArrayRef = concat_arrays(&anys).ok()?;
2783        let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2784        Some(std::sync::Arc::new(ba))
2785    }
2786}
2787
2788#[derive(Debug, Clone)]
2789pub struct LayerInfo {
2790    pub vertex_count: usize,
2791    pub parallel_eligible: bool,
2792    pub sample_cells: Vec<String>, // Sample of up to 5 cell addresses
2793}
2794
2795#[derive(Debug, Clone)]
2796pub struct EvalPlan {
2797    pub total_vertices_to_evaluate: usize,
2798    pub layers: Vec<LayerInfo>,
2799    pub cycles_detected: usize,
2800    pub dirty_count: usize,
2801    pub volatile_count: usize,
2802    pub parallel_enabled: bool,
2803    pub estimated_parallel_layers: usize,
2804    pub target_cells: Vec<String>,
2805}
2806
2807/// Whether the configured FormulaPlane mode is ignored (spans never placed):
2808/// always, since the dependency authority is the runtime path (design §10).
2809#[inline]
2810fn plane_mode_ignored() -> bool {
2811    true
2812}
2813
2814/// Test-support probe for sibling-crate tests: true when span placement is
2815/// off because the FormulaPlane mode is ignored (see `plane_mode_ignored`).
2816#[cfg(feature = "test-support")]
2817#[doc(hidden)]
2818pub fn formula_plane_mode_ignored_for_test() -> bool {
2819    plane_mode_ignored()
2820}
2821
2822impl<R> Engine<R>
2823where
2824    R: EvaluationContext,
2825{
2826    /// # Panics
2827    /// Panics when `config.cycle` is invalid ([`CycleConfig::validate`],
2828    /// spec §2): `Iterate` with `detection: Static`, `max_iterations == 0`,
2829    /// or a negative/non-finite `max_change`. `EvalConfig::with_cycle`
2830    /// rejects these at build; this re-validates configs assembled via
2831    /// struct literals.
2832    pub fn new(resolver: R, config: EvalConfig) -> Self {
2833        // Under the unified authority the FormulaPlane mode is accepted and
2834        // ignored (design §10). Normalizing the stored mode keeps external
2835        // readers of `config` (loaders choosing a span-preparation route) on
2836        // the per-cell path; engine reads go through `formula_plane_mode()`.
2837        let config = if plane_mode_ignored() {
2838            EvalConfig {
2839                formula_plane_mode: FormulaPlaneMode::Off,
2840                ..config
2841            }
2842        } else {
2843            config
2844        };
2845        if let Err(msg) = config.cycle.validate() {
2846            panic!("invalid CycleConfig: {msg}");
2847        }
2848        crate::builtins::load_builtins();
2849        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2850            &config.evaluation_budgets,
2851            config.max_vertices,
2852            config.max_memory_mb,
2853            config.max_eval_time,
2854        );
2855
2856        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2857            #[cfg(feature = "system-clock")]
2858            {
2859                Arc::new(crate::timezone::SystemClock::new(
2860                    crate::timezone::TimeZoneSpec::default(),
2861                ))
2862            }
2863            #[cfg(not(feature = "system-clock"))]
2864            {
2865                Arc::new(crate::timezone::FixedClock::new(
2866                    chrono::DateTime::UNIX_EPOCH,
2867                    crate::timezone::TimeZoneSpec::Utc,
2868                ))
2869            }
2870        });
2871
2872        // Initialize thread pool based on config
2873        let thread_pool = if config.enable_parallel {
2874            let mut builder = ThreadPoolBuilder::new();
2875            if let Some(max_threads) = config.max_threads {
2876                builder = builder.num_threads(max_threads);
2877            }
2878
2879            match builder.build() {
2880                Ok(pool) => Some(Arc::new(pool)),
2881                Err(_) => {
2882                    // Fall back to sequential evaluation if thread pool creation fails
2883                    None
2884                }
2885            }
2886        } else {
2887            None
2888        };
2889
2890        // C1a retained/cache budgets are observational; cache defaults stay explicit.
2891        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2892        let function_provider_revision_seen = resolver.planning_semantic_revision();
2893        let mut engine = Self {
2894            criteria_mask_cache: Default::default(),
2895            graph: DependencyGraph::new_with_config(config.clone()),
2896            resolver,
2897            resolved_functions: Default::default(),
2898            resolved_sheet_names: FxHashMap::default(),
2899            resolution_sheet_revision: None,
2900            resolution_arrow_sheet_names: Vec::new(),
2901            resolved_arrow_sheets: Vec::new(),
2902            arrow_sheet_index: std::sync::OnceLock::new(),
2903            config,
2904            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2905            clock: crate::timezone::SnapshotClock::new(clock),
2906            thread_pool,
2907            recalc_epoch: 0,
2908            snapshot_id: std::sync::atomic::AtomicU64::new(1),
2909            topology_epoch: 0,
2910            cached_static_schedule: None,
2911            recent_schedules: Vec::new(),
2912            base_schedule: None,
2913            #[cfg(any(test, feature = "benchmark_internal"))]
2914            recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
2915            spill_mgr: ShimSpillManager::default(),
2916            arrow_sheets: SheetStore::default(),
2917            format_registry: crate::format::FormatRegistry::default(),
2918            derived_formats: Default::default(),
2919            #[cfg(test)]
2920            derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
2921            #[cfg(test)]
2922            family_members_for_test: std::sync::atomic::AtomicU64::new(0),
2923            #[cfg(test)]
2924            invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
2925            #[cfg(test)]
2926            lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
2927            #[cfg(test)]
2928            chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
2929            #[cfg(test)]
2930            lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
2931            #[cfg(test)]
2932            criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
2933            #[cfg(test)]
2934            memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
2935            compressed_at_build: None,
2936            #[cfg(test)]
2937            computed_overlay_set_explicit_entry_operations_for_test: 0,
2938            #[cfg(test)]
2939            computed_overlay_stale_clear_range_effects_for_test: 0,
2940            #[cfg(test)]
2941            computed_overlay_stale_clear_offset_attempts_for_test: 0,
2942            #[cfg(test)]
2943            computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
2944            has_edited: false,
2945            overlay_compactions: 0,
2946            computed_overlay_bytes_estimate: 0,
2947            computed_overlay_mirroring_disabled: false,
2948            force_materialize_range_views: false,
2949            row_bounds_cache: std::sync::RwLock::new(None),
2950            used_axis_bounds_cache: std::sync::RwLock::new(None),
2951            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2952            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2953            source_formula_token: Arc::new(()),
2954            source_family_mode: crate::engine::SourceFamilyMode::from_env(),
2955            source_family_counters: crate::engine::SourceFamilyCounters::default(),
2956            recalc_plan_token: Arc::new(()),
2957            staged_formulas: std::collections::HashMap::new(),
2958            staged_formula_index: StagedFormulaIndex::default(),
2959            blocked_pending_spills: Vec::new(),
2960            spill_blocker_waiters: SpillBlockerWaiters::default(),
2961            row_visibility: FxHashMap::default(),
2962            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2963            formula_parse_diagnostics: Vec::new(),
2964            last_formula_ingest_report: None,
2965            formula_ingest_report_total: FormulaIngestReport::default(),
2966            active_cancel_flag: None,
2967            active_evaluation_deadline: None,
2968            action_depth: 0,
2969            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2970            virtual_dep_fallback_activations: 0,
2971            last_cycle_telemetry: CycleTelemetry::default(),
2972            next_evaluation_resource_request_id: 1,
2973            evaluation_resource_request_depth: 0,
2974            active_evaluation_resource_request: None,
2975            last_evaluation_resource_request: None,
2976            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2977            evaluation_resource_request_started_at: None,
2978            evaluation_resource_budgets: resolved_resources.budgets,
2979            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2980            active_resource_ledger: None,
2981            source_cache_footprints: Vec::new(),
2982            source_cache_accounted: 0,
2983            pending_iterative_redirty: Vec::new(),
2984            snapshot_evaluation_active: false,
2985            snapshot_volatile_redirty_pending: false,
2986            retained_scc_members: FxHashMap::default(),
2987            next_retained_scc_id: 0,
2988            retained_scc_config_fingerprint: 0,
2989            retained_scc_function_epoch_seen: 0,
2990            retained_scc_provider_revision_seen: None,
2991            retained_scc_dirty_at_begin: Vec::new(),
2992            iterative_state_values: FxHashMap::default(),
2993            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2994            function_provider_revision_seen,
2995            #[cfg(feature = "tracing")]
2996            trace_evaluation_counters: TraceEvaluationCounters::default(),
2997            #[cfg(test)]
2998            evaluation_request_begin_count_for_test: 0,
2999            #[cfg(any(test, feature = "test-support"))]
3000            before_prepared_span_commit_hook: None,
3001            #[cfg(test)]
3002            before_target_preparation_commit_hook: None,
3003            #[cfg(test)]
3004            before_target_planning_snapshot_hook: None,
3005            #[cfg(test)]
3006            inject_target_semantic_stale_once_for_test: false,
3007            #[cfg(test)]
3008            force_virtual_dep_changes_remaining_for_test: 0,
3009            freshness: Default::default(),
3010            #[cfg(test)]
3011            fail_evaluation_commit_preflight_once_for_test: false,
3012            #[cfg(test)]
3013            target_preparation_fault_for_test: None,
3014            #[cfg(test)]
3015            force_non_cycle_schedule_fallback_for_test: false,
3016            #[cfg(test)]
3017            before_legacy_fallback_final_provider_sample_hook: None,
3018            #[cfg(test)]
3019            after_eager_proposal_commit_hook: None,
3020        };
3021        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
3022        engine.config.arrow_storage_enabled = true;
3023        engine.config.delta_overlay_enabled = true;
3024        engine.config.write_formula_overlay_enabled = true;
3025        let default_sheet = engine.graph.default_sheet_name().to_string();
3026        engine.ensure_arrow_sheet(&default_sheet);
3027        engine
3028    }
3029
3030    /// Create an Engine with a custom thread pool (for shared thread pool scenarios)
3031    ///
3032    /// # Panics
3033    /// Panics when `config.cycle` is invalid, exactly like [`Engine::new`].
3034    pub fn with_thread_pool(
3035        resolver: R,
3036        config: EvalConfig,
3037        thread_pool: Arc<rayon::ThreadPool>,
3038    ) -> Self {
3039        if let Err(msg) = config.cycle.validate() {
3040            panic!("invalid CycleConfig: {msg}");
3041        }
3042        crate::builtins::load_builtins();
3043        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
3044            &config.evaluation_budgets,
3045            config.max_vertices,
3046            config.max_memory_mb,
3047            config.max_eval_time,
3048        );
3049        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
3050            #[cfg(feature = "system-clock")]
3051            {
3052                Arc::new(crate::timezone::SystemClock::new(
3053                    crate::timezone::TimeZoneSpec::default(),
3054                ))
3055            }
3056            #[cfg(not(feature = "system-clock"))]
3057            {
3058                Arc::new(crate::timezone::FixedClock::new(
3059                    chrono::DateTime::UNIX_EPOCH,
3060                    crate::timezone::TimeZoneSpec::Utc,
3061                ))
3062            }
3063        });
3064        // C1a retained/cache budgets are observational; cache defaults stay explicit.
3065        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
3066        let function_provider_revision_seen = resolver.planning_semantic_revision();
3067        let mut engine = Self {
3068            criteria_mask_cache: Default::default(),
3069            graph: DependencyGraph::new_with_config(config.clone()),
3070            resolver,
3071            resolved_functions: Default::default(),
3072            resolved_sheet_names: FxHashMap::default(),
3073            resolution_sheet_revision: None,
3074            resolution_arrow_sheet_names: Vec::new(),
3075            resolved_arrow_sheets: Vec::new(),
3076            arrow_sheet_index: std::sync::OnceLock::new(),
3077            config,
3078            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
3079            clock: crate::timezone::SnapshotClock::new(clock),
3080            thread_pool: Some(thread_pool),
3081            recalc_epoch: 0,
3082            snapshot_id: std::sync::atomic::AtomicU64::new(1),
3083            topology_epoch: 0,
3084            cached_static_schedule: None,
3085            recent_schedules: Vec::new(),
3086            base_schedule: None,
3087            #[cfg(any(test, feature = "benchmark_internal"))]
3088            recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
3089            spill_mgr: ShimSpillManager::default(),
3090            arrow_sheets: SheetStore::default(),
3091            format_registry: crate::format::FormatRegistry::default(),
3092            derived_formats: Default::default(),
3093            #[cfg(test)]
3094            derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
3095            #[cfg(test)]
3096            family_members_for_test: std::sync::atomic::AtomicU64::new(0),
3097            #[cfg(test)]
3098            invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
3099            #[cfg(test)]
3100            lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
3101            #[cfg(test)]
3102            chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
3103            #[cfg(test)]
3104            lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
3105            #[cfg(test)]
3106            criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
3107            #[cfg(test)]
3108            memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
3109            compressed_at_build: None,
3110            #[cfg(test)]
3111            computed_overlay_set_explicit_entry_operations_for_test: 0,
3112            #[cfg(test)]
3113            computed_overlay_stale_clear_range_effects_for_test: 0,
3114            #[cfg(test)]
3115            computed_overlay_stale_clear_offset_attempts_for_test: 0,
3116            #[cfg(test)]
3117            computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
3118            has_edited: false,
3119            overlay_compactions: 0,
3120            computed_overlay_bytes_estimate: 0,
3121            computed_overlay_mirroring_disabled: false,
3122            force_materialize_range_views: false,
3123            row_bounds_cache: std::sync::RwLock::new(None),
3124            used_axis_bounds_cache: std::sync::RwLock::new(None),
3125            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
3126            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
3127            source_formula_token: Arc::new(()),
3128            source_family_mode: crate::engine::SourceFamilyMode::from_env(),
3129            source_family_counters: crate::engine::SourceFamilyCounters::default(),
3130            recalc_plan_token: Arc::new(()),
3131            staged_formulas: std::collections::HashMap::new(),
3132            staged_formula_index: StagedFormulaIndex::default(),
3133            blocked_pending_spills: Vec::new(),
3134            spill_blocker_waiters: SpillBlockerWaiters::default(),
3135            row_visibility: FxHashMap::default(),
3136            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
3137            formula_parse_diagnostics: Vec::new(),
3138            last_formula_ingest_report: None,
3139            formula_ingest_report_total: FormulaIngestReport::default(),
3140            active_cancel_flag: None,
3141            active_evaluation_deadline: None,
3142            action_depth: 0,
3143            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
3144            virtual_dep_fallback_activations: 0,
3145            last_cycle_telemetry: CycleTelemetry::default(),
3146            next_evaluation_resource_request_id: 1,
3147            evaluation_resource_request_depth: 0,
3148            active_evaluation_resource_request: None,
3149            last_evaluation_resource_request: None,
3150            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
3151            evaluation_resource_request_started_at: None,
3152            evaluation_resource_budgets: resolved_resources.budgets,
3153            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
3154            active_resource_ledger: None,
3155            source_cache_footprints: Vec::new(),
3156            source_cache_accounted: 0,
3157            pending_iterative_redirty: Vec::new(),
3158            snapshot_evaluation_active: false,
3159            snapshot_volatile_redirty_pending: false,
3160            retained_scc_members: FxHashMap::default(),
3161            next_retained_scc_id: 0,
3162            retained_scc_config_fingerprint: 0,
3163            retained_scc_function_epoch_seen: 0,
3164            retained_scc_provider_revision_seen: None,
3165            retained_scc_dirty_at_begin: Vec::new(),
3166            iterative_state_values: FxHashMap::default(),
3167            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
3168            function_provider_revision_seen,
3169            #[cfg(feature = "tracing")]
3170            trace_evaluation_counters: TraceEvaluationCounters::default(),
3171            #[cfg(test)]
3172            evaluation_request_begin_count_for_test: 0,
3173            #[cfg(any(test, feature = "test-support"))]
3174            before_prepared_span_commit_hook: None,
3175            #[cfg(test)]
3176            before_target_preparation_commit_hook: None,
3177            #[cfg(test)]
3178            before_target_planning_snapshot_hook: None,
3179            #[cfg(test)]
3180            inject_target_semantic_stale_once_for_test: false,
3181            #[cfg(test)]
3182            force_virtual_dep_changes_remaining_for_test: 0,
3183            freshness: Default::default(),
3184            #[cfg(test)]
3185            fail_evaluation_commit_preflight_once_for_test: false,
3186            #[cfg(test)]
3187            target_preparation_fault_for_test: None,
3188            #[cfg(test)]
3189            force_non_cycle_schedule_fallback_for_test: false,
3190            #[cfg(test)]
3191            before_legacy_fallback_final_provider_sample_hook: None,
3192            #[cfg(test)]
3193            after_eager_proposal_commit_hook: None,
3194        };
3195        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
3196        engine.config.arrow_storage_enabled = true;
3197        engine.config.delta_overlay_enabled = true;
3198        engine.config.write_formula_overlay_enabled = true;
3199        let default_sheet = engine.graph.default_sheet_name().to_string();
3200        engine.ensure_arrow_sheet(&default_sheet);
3201        engine
3202    }
3203
3204    pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
3205        &self.workbook_load_limits
3206    }
3207
3208    pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
3209        self.workbook_load_limits = limits;
3210    }
3211
3212    fn clear_source_cache(&self) {
3213        if let Ok(mut g) = self.source_cache.write() {
3214            *g = SourceCache::default();
3215        }
3216    }
3217
3218    pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
3219        &self.last_virtual_dep_telemetry
3220    }
3221
3222    /// Telemetry from Runtime SCC evaluation during the most recent
3223    /// evaluation request (always default-zero under `CycleDetection::Static`
3224    /// or when `enable_virtual_dep_telemetry` is off).
3225    pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
3226        &self.last_cycle_telemetry
3227    }
3228
3229    /// Resource observations for the most recently completed public evaluation request.
3230    pub fn last_evaluation_resource_request_stats(
3231        &self,
3232    ) -> Option<&EvaluationResourceRequestStats> {
3233        self.last_evaluation_resource_request.as_ref()
3234    }
3235
3236    /// Cumulative resource observations since engine creation or the last telemetry reset.
3237    pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
3238        self.evaluation_resource_baseline
3239    }
3240
3241    pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
3242        &self.evaluation_resource_budgets
3243    }
3244
3245    /// At most one diagnostic is emitted for deprecated resource fields.
3246    pub fn evaluation_resource_config_diagnostic(
3247        &self,
3248    ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
3249        self.evaluation_resource_config_diagnostic.as_ref()
3250    }
3251
3252    /// Reset accumulated and last-request observations without reusing request IDs.
3253    pub fn reset_evaluation_resource_telemetry(&mut self) {
3254        self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
3255        self.last_evaluation_resource_request = None;
3256    }
3257
3258    // Reconcile without replay locks: several packages may share the same Arc.
3259    // Weak tokens neither retain dead packages nor duplicate their allocations.
3260    fn reconcile_source_cache_footprints(&mut self) -> Result<(), ExcelError> {
3261        self.blocked_pending_spills.retain(|&(vertex, anchor, _)| {
3262            self.graph.vertex_exists(vertex)
3263                && self.graph.get_cell_ref(vertex) == Some(anchor)
3264                && matches!(
3265                    self.graph.get_vertex_kind(vertex),
3266                    VertexKind::FormulaScalar | VertexKind::FormulaArray
3267                )
3268        });
3269        if self.blocked_pending_spills.is_empty() {
3270            self.blocked_pending_spills = Vec::new();
3271        }
3272        let mut bytes = (self.blocked_pending_spills.capacity()
3273            * std::mem::size_of::<(VertexId, CellRef, Region)>()) as u64;
3274        self.source_cache_footprints.retain(|weak| {
3275            let Some(footprint) = weak.upgrade() else {
3276                return false;
3277            };
3278            bytes = bytes.saturating_add(footprint.load(std::sync::atomic::Ordering::Acquire));
3279            true
3280        });
3281        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3282            ledger
3283                .release_retained(self.source_cache_accounted)
3284                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3285            // Observe first: even a tightened budget cannot erase live ownership.
3286            ledger.observe_retained(bytes);
3287            self.source_cache_accounted = bytes;
3288            ledger
3289                .reserve_retained(0)
3290                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3291        }
3292        Ok(())
3293    }
3294
3295    fn duration_ns(duration: std::time::Duration) -> u64 {
3296        u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
3297    }
3298
3299    fn observe_evaluation_resource_request<T>(
3300        &mut self,
3301        kind: EvaluationRequestKind,
3302        evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3303    ) -> Result<T, ExcelError> {
3304        let outermost = self.evaluation_resource_request_depth == 0;
3305        if outermost && self.snapshot_volatile_redirty_pending {
3306            self.snapshot_volatile_redirty_pending = false;
3307            self.graph.redirty_volatiles();
3308        }
3309        #[cfg(feature = "tracing")]
3310        if outermost {
3311            self.trace_evaluation_counters = TraceEvaluationCounters::default();
3312        }
3313        #[cfg(feature = "tracing")]
3314        let request_kind = if matches!(
3315            kind,
3316            EvaluationRequestKind::Full
3317                | EvaluationRequestKind::FullWithDelta
3318                | EvaluationRequestKind::FullCancellable
3319                | EvaluationRequestKind::FullLogged
3320        ) {
3321            "full"
3322        } else {
3323            "targeted"
3324        };
3325        #[cfg(feature = "tracing")]
3326        let _request_span = outermost.then(|| {
3327            crate::engine::trace::fz_span!(
3328                tracing::Level::INFO,
3329                "evaluate",
3330                "evaluate.request",
3331                kind = request_kind,
3332                mode = ?FormulaPlaneMode::Off
3333            )
3334        });
3335        if outermost {
3336            let request_id = self.next_evaluation_resource_request_id;
3337            self.next_evaluation_resource_request_id = request_id
3338                .checked_add(1)
3339                .expect("evaluation resource request ID exhausted");
3340            self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
3341                request_id,
3342                kind,
3343                FormulaPlaneMode::Off,
3344                self.staged_formula_count(),
3345            ));
3346            self.evaluation_resource_baseline.record_started(request_id);
3347            self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
3348            self.source_cache_accounted = 0;
3349            self.active_resource_ledger = Some(ResourceLedger::new(
3350                Some(request_id),
3351                self.evaluation_resource_budgets.clone(),
3352            ));
3353        }
3354        self.evaluation_resource_request_depth =
3355            self.evaluation_resource_request_depth.saturating_add(1);
3356        let result = if outermost {
3357            self.reconcile_source_cache_footprints()
3358                .and_then(|()| self.resource_checkpoint(0))
3359                .and_then(|()| evaluate(self))
3360        } else {
3361            evaluate(self)
3362        };
3363        if outermost && result.is_err() {
3364            self.freshness_abort_pass();
3365        }
3366        self.evaluation_resource_request_depth =
3367            self.evaluation_resource_request_depth.saturating_sub(1);
3368
3369        let reconciliation = if outermost {
3370            self.reconcile_source_cache_footprints()
3371        } else {
3372            Ok(())
3373        };
3374        let result = result.and_then(|value| reconciliation.map(|()| value));
3375        if outermost {
3376            let total_ns = self
3377                .evaluation_resource_request_started_at
3378                .take()
3379                .map(|start| Self::duration_ns(start.elapsed()))
3380                .unwrap_or(0);
3381            let mut stats = self
3382                .active_evaluation_resource_request
3383                .take()
3384                .expect("outer evaluation resource request has active stats");
3385            let mut ledger = self
3386                .active_resource_ledger
3387                .take()
3388                .expect("outer evaluation resource request has active ledger");
3389            ledger.release_all_scratch();
3390            stats.ledger.update(ledger.snapshot());
3391            stats.outcome = match &result {
3392                Ok(_) => EvaluationRequestOutcome::Success,
3393                Err(error) if error.kind == ExcelErrorKind::Cancelled => {
3394                    EvaluationRequestOutcome::Cancelled
3395                }
3396                Err(_) => EvaluationRequestOutcome::Error,
3397            };
3398            if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
3399                stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
3400                    FormulaDirtyLeaseOutcome::RetainedOnCancellation
3401                } else {
3402                    FormulaDirtyLeaseOutcome::RetainedOnError
3403                };
3404            }
3405            stats.phases.total_ns = total_ns;
3406            let attributed = stats
3407                .phases
3408                .staged_prepare_ns
3409                .saturating_add(stats.phases.topology_ns)
3410                .saturating_add(stats.phases.materialization_ns);
3411            stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
3412            self.evaluation_resource_baseline.record_finished(&stats);
3413            self.last_evaluation_resource_request = Some(stats);
3414            crate::engine::trace::fz_event!(
3415                tracing::Level::INFO,
3416                "evaluate",
3417                "evaluate.summary",
3418                computed_vertices = self.trace_evaluation_counters.computed_vertices,
3419                cycles = self.trace_evaluation_counters.cycles,
3420                cancelled = matches!(
3421                    &result,
3422                    Err(error) if error.kind == ExcelErrorKind::Cancelled
3423                )
3424            );
3425        }
3426        result
3427    }
3428
3429    pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
3430        self.evaluation_resource_budgets = budgets.clone();
3431        self.config.evaluation_budgets = budgets.clone();
3432        self.graph.set_evaluation_budgets(budgets);
3433    }
3434
3435    #[cfg(test)]
3436    pub(crate) fn set_evaluation_budgets_for_test(
3437        &mut self,
3438        budgets: crate::engine::EvaluationBudgets,
3439    ) {
3440        self.set_evaluation_resource_budgets(budgets);
3441    }
3442
3443    fn preflight_evaluation_commit_window(
3444        &mut self,
3445        bounded_writes: usize,
3446    ) -> Result<crate::instant::FzInstant, ExcelError> {
3447        #[cfg(test)]
3448        if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
3449            return Err(crate::engine::ResourceLedgerError::Exhausted(
3450                formualizer_common::ResourceExhaustionDetail {
3451                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3452                    limit: 0,
3453                    observed: 1,
3454                    request_id: self
3455                        .active_evaluation_resource_request
3456                        .as_ref()
3457                        .map(|stats| stats.request_id),
3458                },
3459            )
3460            .into_excel_error());
3461        }
3462        let estimate = std::time::Duration::from_nanos(
3463            u64::try_from(bounded_writes)
3464                .unwrap_or(u64::MAX)
3465                .saturating_mul(100),
3466        );
3467        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3468            ledger
3469                .preflight_commit_window(estimate)
3470                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3471        }
3472        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3473            stats.evaluation_commit_preflight_count =
3474                stats.evaluation_commit_preflight_count.saturating_add(1);
3475            stats.evaluation_commit_estimated_ns = stats
3476                .evaluation_commit_estimated_ns
3477                .saturating_add(Self::duration_ns(estimate));
3478        }
3479        Ok(crate::instant::FzInstant::now())
3480    }
3481
3482    fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3483        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3484            stats.evaluation_commit_actual_ns = stats
3485                .evaluation_commit_actual_ns
3486                .saturating_add(Self::duration_ns(started.elapsed()));
3487        }
3488    }
3489
3490    /// Post-work cancellation boundary: call after evaluating a unit (or a
3491    /// parallel group) and before committing it. A function that observed
3492    /// the request's token returns `Err(Cancelled)`, which the evaluator
3493    /// turns into a `#CANCELLED` value; any other result computed across the
3494    /// signal is equally not a finished result. Neither may publish: the
3495    /// caller returns this error before committing, so the unit stays dirty,
3496    /// the request reports `Cancelled`, and a failed pass restores the
3497    /// vertices it already committed (`freshness_abort_pass`). The token is
3498    /// the discriminator: a `#CANCELLED` value with no live cancellation is
3499    /// ordinary data and commits. Deadlines are not checked here; finished
3500    /// work is not discarded on a deadline.
3501    fn live_cancellation_after_work(&self, message: &'static str) -> Result<(), ExcelError> {
3502        if self
3503            .active_cancel_flag
3504            .as_ref()
3505            .is_some_and(|cancel| cancel.is_cancelled())
3506        {
3507            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3508        }
3509        Ok(())
3510    }
3511
3512    fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3513        if self
3514            .active_cancel_flag
3515            .as_ref()
3516            .is_some_and(|cancel| cancel.is_cancelled())
3517        {
3518            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3519        }
3520        if self
3521            .active_evaluation_deadline
3522            .is_some_and(|deadline| Instant::now() >= deadline)
3523        {
3524            return Err(crate::engine::ResourceLedgerError::Exhausted(
3525                formualizer_common::ResourceExhaustionDetail {
3526                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3527                    limit: 0,
3528                    observed: 1,
3529                    request_id: self
3530                        .active_evaluation_resource_request
3531                        .as_ref()
3532                        .map(|request| request.request_id),
3533                },
3534            )
3535            .into_excel_error()
3536            .with_message(message));
3537        }
3538        Ok(())
3539    }
3540
3541    fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3542        let Some(ledger) = self.active_resource_ledger.as_mut() else {
3543            return Ok(());
3544        };
3545        ledger
3546            .charge_work(work_units)
3547            .and_then(|()| ledger.checkpoint_deadline())
3548            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3549    }
3550
3551    fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3552        if work_units == 0 {
3553            return self.resource_checkpoint(0);
3554        }
3555        while work_units > 0 {
3556            let chunk = work_units.min(256);
3557            self.resource_checkpoint(chunk)?;
3558            work_units -= chunk;
3559        }
3560        Ok(())
3561    }
3562
3563    fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3564        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3565            // Exact request topology and activation of the scratch cap are C1b. C1a records
3566            // scoped ownership but must not introduce a new skip or terminal path.
3567            ledger.observe_scratch(bytes);
3568        }
3569        Ok(())
3570    }
3571
3572    fn release_request_scratch(&mut self, bytes: u64) {
3573        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3574            let released = ledger.release_scratch(bytes);
3575            debug_assert!(
3576                released.is_ok(),
3577                "request scratch release exceeded the outstanding reservation"
3578            );
3579        }
3580    }
3581
3582    fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3583        self.active_resource_ledger
3584            .as_mut()
3585            .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3586            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3587    }
3588
3589    fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3590        self.active_resource_ledger
3591            .as_mut()
3592            .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3593            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3594    }
3595
3596    fn with_request_scratch<T>(
3597        &mut self,
3598        bytes: u64,
3599        work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3600    ) -> Result<T, ExcelError> {
3601        self.reserve_request_scratch(bytes)?;
3602        let result = work(self);
3603        self.release_request_scratch(bytes);
3604        result
3605    }
3606
3607    fn observe_staged_preparation(
3608        &mut self,
3609        selected: usize,
3610        retained: usize,
3611        elapsed: std::time::Duration,
3612    ) {
3613        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3614            stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3615            stats.staged_retained = retained as u64;
3616            stats.phases.staged_prepare_ns = stats
3617                .phases
3618                .staged_prepare_ns
3619                .saturating_add(Self::duration_ns(elapsed));
3620        }
3621    }
3622
3623    fn graph_admission_enabled(&self) -> bool {
3624        crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3625    }
3626
3627    fn preflight_graph_admission(
3628        &mut self,
3629        usage: crate::engine::resource_ledger::GraphAdmission,
3630    ) -> Result<(), ExcelError> {
3631        let request_id = self
3632            .active_evaluation_resource_request
3633            .as_ref()
3634            .map(|stats| stats.request_id);
3635        crate::engine::resource_ledger::preflight_graph_admission(
3636            &self.evaluation_resource_budgets,
3637            usage,
3638            request_id,
3639        )
3640        .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3641    }
3642
3643    fn prepared_legacy_admission(
3644        &mut self,
3645        plan: &PreparedLegacyGraphPlan,
3646        materialization_cells: u64,
3647    ) -> Result<(), ExcelError> {
3648        if !self.graph_admission_enabled() {
3649            return Ok(());
3650        }
3651        let stats = self.graph.baseline_stats();
3652        let added_edges = plan.planned_edge_count().ok_or_else(|| {
3653            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3654        })?;
3655        let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3656            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3657        })?;
3658        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3659            final_vertices: stats
3660                .graph_vertex_count
3661                .checked_add(plan.new_vertex_count())
3662                .ok_or_else(|| {
3663                    ExcelError::new(ExcelErrorKind::NImpl)
3664                        .with_message("graph vertex count overflow")
3665                })?,
3666            final_edges: stats
3667                .graph_edge_count
3668                .checked_sub(removed_edges)
3669                .and_then(|count| count.checked_add(added_edges))
3670                .ok_or_else(|| {
3671                    ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3672                })?,
3673            materialization_cells,
3674            added_vertices: plan.new_vertex_count(),
3675            added_edges,
3676        })
3677    }
3678
3679    fn observe_target_admission_failure(
3680        &mut self,
3681        reason: formualizer_common::ResourceExhaustionReason,
3682    ) {
3683        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3684            stats.target_admission_failure = Some(reason);
3685        }
3686    }
3687
3688    fn observe_target_preparation_report(
3689        &mut self,
3690        report: &crate::engine::PreparedTargetGraphReport,
3691    ) {
3692        let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3693            let index = match reason {
3694                crate::engine::OpaqueReason::DynamicReference => 0,
3695                crate::engine::OpaqueReason::RuntimeTextReference => 1,
3696                crate::engine::OpaqueReason::UnknownFunction => 2,
3697                crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3698                crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3699                crate::engine::OpaqueReason::UnresolvedName => 5,
3700                crate::engine::OpaqueReason::UnresolvedTable => 6,
3701                crate::engine::OpaqueReason::FormulaName => 7,
3702                crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3703                crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3704                crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3705            };
3706            1u64 << index
3707        };
3708        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3709            stats.staged_selected = report.selected_staged_cells as u64;
3710            stats.staged_retained = report.retained_staged_cells as u64;
3711            stats.target_requested = report.requested_targets as u64;
3712            stats.target_normalized_regions = report.normalized_regions as u64;
3713            stats.target_scope_level = match &report.widened_scope {
3714                crate::engine::PrepareScope::Exact => 0,
3715                crate::engine::PrepareScope::Sheets(_) => 1,
3716                crate::engine::PrepareScope::Workbook => 2,
3717            };
3718            stats.target_widening_reason_bits = report
3719                .widening_reasons
3720                .iter()
3721                .copied()
3722                .fold(0, |bits, reason| bits | reason_bit(reason));
3723            stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3724            stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3725            stats.target_commit_estimated_work = report.estimated_commit_work;
3726            stats.target_commit_actual_work = report.actual_commit_work;
3727            stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3728            stats.phases.staged_prepare_ns = stats
3729                .phases
3730                .staged_prepare_ns
3731                .saturating_add(Self::duration_ns(report.commit_window));
3732        }
3733    }
3734
3735    /// Begin a new evaluation request: reset per-recalc cycle telemetry and
3736    /// take the per-recalc volatile clock sample. Called at the start of
3737    /// every evaluation request that walks schedule units.
3738    /// The per-request function table when it describes the current registry,
3739    /// provider and arena; `None` outside a request when it may be stale.
3740    pub(crate) fn current_resolved_functions(
3741        &self,
3742    ) -> Option<&crate::interpreter::ResolvedFunctions> {
3743        self.resolved_functions
3744            .is_current(self.graph.data_store(), &self.resolver)
3745            .then_some(&self.resolved_functions)
3746    }
3747
3748    fn refresh_resolution_snapshots(&mut self) {
3749        self.resolved_functions
3750            .refresh(self.graph.data_store(), &self.resolver);
3751        let named_asts = self
3752            .graph
3753            .named_ranges_iter()
3754            .map(|(_, named)| named)
3755            .chain(self.graph.sheet_named_ranges_iter().map(|(_, named)| named))
3756            .filter_map(|named| match &named.definition {
3757                NamedDefinition::Formula { ast, .. } => Some(ast),
3758                _ => None,
3759            });
3760        self.resolved_functions.refresh_ast_functions(
3761            self.graph.data_store(),
3762            &self.resolver,
3763            named_asts,
3764            self.graph.symbol_revision(),
3765        );
3766        // Arrow sheet storage is publicly mutable. Compare names and order
3767        // exactly (including same-length replacements) without allocating on
3768        // unchanged requests; a fingerprint alone could silently collide.
3769        let revision = (
3770            self.graph.sheet_reg().revision(),
3771            self.graph.data_store().resolution_revision(),
3772            self.graph.data_store().sheet_names().count(),
3773        );
3774        let arrow_names_unchanged = self
3775            .resolution_arrow_sheet_names
3776            .iter()
3777            .map(|name| name.as_ref())
3778            .eq(self
3779                .arrow_sheets
3780                .sheets
3781                .iter()
3782                .map(|sheet| sheet.name.as_ref()));
3783        if self.resolution_sheet_revision == Some(revision) && arrow_names_unchanged {
3784            return;
3785        }
3786        self.resolution_arrow_sheet_names.clear();
3787        self.resolution_arrow_sheet_names.extend(
3788            self.arrow_sheets
3789                .sheets
3790                .iter()
3791                .map(|sheet| Arc::clone(&sheet.name)),
3792        );
3793        self.resolved_sheet_names.clear();
3794        for id in self.graph.data_store().sheet_names() {
3795            let name = self.graph.data_store().resolve_ast_string(id);
3796            if let Some(sheet_id) = self.graph.sheet_id(name) {
3797                self.resolved_sheet_names.insert(
3798                    id,
3799                    (
3800                        sheet_id,
3801                        self.arrow_sheets
3802                            .sheets
3803                            .iter()
3804                            .position(|sheet| sheet.name.as_ref() == name),
3805                    ),
3806                );
3807            }
3808        }
3809        self.resolved_arrow_sheets.clear();
3810        for (index, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
3811            if let Some(id) = self.graph.sheet_id(&sheet.name) {
3812                let len = usize::from(id).saturating_add(1);
3813                if self.resolved_arrow_sheets.len() < len {
3814                    self.resolved_arrow_sheets.resize(len, None);
3815                }
3816                if let Some(slot) = self.resolved_arrow_sheets.get_mut(usize::from(id)) {
3817                    *slot = Some(index);
3818                }
3819            }
3820        }
3821        self.resolution_sheet_revision = Some(revision);
3822    }
3823
3824    fn begin_evaluation_request(&mut self) {
3825        self.refresh_resolution_snapshots();
3826        self.freshness_begin_request();
3827        #[cfg(test)]
3828        {
3829            self.evaluation_request_begin_count_for_test = self
3830                .evaluation_request_begin_count_for_test
3831                .saturating_add(1);
3832        }
3833        self.last_cycle_telemetry = CycleTelemetry::default();
3834        self.graph.authority_sync();
3835        // Defensive: consumed at the end of the previous request; a request
3836        // that errored out mid-walk must not leak its members into this one.
3837        self.pending_iterative_redirty.clear();
3838        self.reconcile_retained_sccs_at_request_begin();
3839        // Spec §7.11: NOW()/TODAY() sample the clock ONCE per recalc; every
3840        // read within this request (including SCC iteration passes) observes
3841        // this sample.
3842        self.clock.refresh();
3843    }
3844
3845    /// End-of-recalc redirty: volatile vertices (as always) plus members of
3846    /// SCCs that iterated this recalc without reaching a retainable fixed
3847    /// point (`CyclePolicy::Iterate`), so circular cells re-evaluate on every
3848    /// recalc exactly like Excel's iterative calculation (spec §4
3849    /// persistence / §7.6 accumulator / §7.11 volatile redirty). Retained
3850    /// SCCs (`retained_scc_members`, #368) are left clean. Replaces the bare
3851    /// `graph.redirty_volatiles()` call at every evaluation-flow exit; must
3852    /// run AFTER the flow's `clear_dirty_flags`.
3853    fn redirty_for_next_recalc(&mut self) {
3854        if self.snapshot_evaluation_active {
3855            self.snapshot_volatile_redirty_pending = true;
3856        } else {
3857            self.graph.redirty_volatiles();
3858        }
3859        let pending = std::mem::take(&mut self.pending_iterative_redirty);
3860        let dirty_at_begin = std::mem::take(&mut self.retained_scc_dirty_at_begin);
3861        for (vertex, scc) in dirty_at_begin {
3862            if self.retained_scc_members.get(&vertex) == Some(&scc) {
3863                // No SCC task claimed this member during the request: the
3864                // cycle dissolved and it evaluated as an ordinary formula,
3865                // or the request never reached it. Either way it is no
3866                // longer a retained fixed point. Its persisted value is
3867                // obsolete only if it actually re-evaluated (clean now).
3868                self.retained_scc_members.remove(&vertex);
3869                if !self.graph.is_dirty(vertex) {
3870                    self.iterative_state_values.remove(&vertex);
3871                }
3872            }
3873        }
3874        if !self.iterative_state_values.is_empty() || !self.retained_scc_members.is_empty() {
3875            let graph = &self.graph;
3876            self.iterative_state_values
3877                .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3878            self.retained_scc_members
3879                .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3880        }
3881        // Refresh the §4-persistence snapshot for members that re-run each
3882        // recalc: these final values survive structural edits that clear the
3883        // computed overlay (the only value home in canonical mode) so the
3884        // next SCC task can re-seed from them (see `iterative_state_values`).
3885        for &vertex in &pending {
3886            if !self.graph.is_live_formula_vertex(vertex) {
3887                continue;
3888            }
3889            if let Some(cell) = self.graph.get_cell_ref(vertex) {
3890                let sheet_name = self.graph.sheet_name(cell.sheet_id);
3891                match self.read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1) {
3892                    Some(value) if !matches!(value, LiteralValue::Empty) => {
3893                        self.iterative_state_values.insert(vertex, value);
3894                    }
3895                    _ => {
3896                        self.iterative_state_values.remove(&vertex);
3897                    }
3898                }
3899            }
3900        }
3901        if !pending.is_empty() {
3902            self.graph.redirty_iterative_members(&pending);
3903        }
3904    }
3905
3906    /// Hash of every `EvalConfig` knob that can change the result of a
3907    /// retained SCC without any edit reaching the dependency graph. The
3908    /// function registry is tracked separately and precisely (see
3909    /// `retained_scc_function_epoch_seen`).
3910    fn retained_scc_config_fingerprint(&self) -> u64 {
3911        use std::hash::{Hash, Hasher};
3912        let mut hasher = rustc_hash::FxHasher::default();
3913        let config = &self.config;
3914        std::mem::discriminant(&config.cycle.detection).hash(&mut hasher);
3915        match config.cycle.policy {
3916            CyclePolicy::Error => 0u8.hash(&mut hasher),
3917            CyclePolicy::Iterate {
3918                max_iterations,
3919                max_change,
3920            } => {
3921                1u8.hash(&mut hasher);
3922                max_iterations.hash(&mut hasher);
3923                max_change.to_bits().hash(&mut hasher);
3924            }
3925        }
3926        config.date_system.hash(&mut hasher);
3927        config.workbook_seed.hash(&mut hasher);
3928        std::mem::discriminant(&config.volatile_level).hash(&mut hasher);
3929        // `DeterministicMode` carries a timestamp and a timezone spec; hash
3930        // its Debug rendering rather than growing its derive set for this.
3931        format!("{:?}", config.deterministic_mode).hash(&mut hasher);
3932        config.range_expansion_limit.hash(&mut hasher);
3933        config.max_open_ended_rows.hash(&mut hasher);
3934        config.max_open_ended_cols.hash(&mut hasher);
3935        hasher.finish()
3936    }
3937
3938    /// Request-begin bookkeeping for retained SCCs (#368): drop deleted
3939    /// vertices, invalidate everything when the config fingerprint moved
3940    /// (marking retained members dirty so their SCC tasks run in this
3941    /// request), and record how many retained SCCs are being reused, i.e.
3942    /// have no dirty member at request begin.
3943    fn reconcile_retained_sccs_at_request_begin(&mut self) {
3944        self.retained_scc_dirty_at_begin.clear();
3945        if self.retained_scc_members.is_empty() {
3946            return;
3947        }
3948        // Deleted vertices and members overwritten with a literal (the
3949        // vertex survives as a value cell) are no longer retained formulas.
3950        let graph = &self.graph;
3951        self.retained_scc_members
3952            .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3953        if self.retained_scc_config_fingerprint() != self.retained_scc_config_fingerprint {
3954            let members: Vec<VertexId> = self.retained_scc_members.keys().copied().collect();
3955            self.retained_scc_members.clear();
3956            self.graph.mark_dirty_many(&members);
3957            return;
3958        }
3959        let changes =
3960            crate::function_registry::semantic_changes_since(self.retained_scc_function_epoch_seen);
3961        let global_changed = changes.epoch != self.retained_scc_function_epoch_seen;
3962        let provider_revision = self.resolver.planning_semantic_revision();
3963        let provider_changed = provider_revision != self.retained_scc_provider_revision_seen;
3964        if global_changed || provider_changed {
3965            let changed: BTreeSet<(String, String)> = changes.keys.into_iter().collect();
3966            let affected: Vec<VertexId> = self
3967                .retained_scc_members
3968                .keys()
3969                .copied()
3970                .filter(|&vertex| {
3971                    let Some(ast) = self.graph.get_formula(vertex) else {
3972                        return true;
3973                    };
3974                    (global_changed
3975                        && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed)))
3976                        || (provider_changed && Self::ast_contains_function(&ast))
3977                })
3978                .collect();
3979            for vertex in &affected {
3980                self.retained_scc_members.remove(vertex);
3981            }
3982            if !affected.is_empty() {
3983                self.graph.mark_dirty_many(&affected);
3984            }
3985            self.retained_scc_function_epoch_seen = changes.epoch;
3986            self.retained_scc_provider_revision_seen = provider_revision;
3987        }
3988        let mut dirty_sccs: FxHashSet<u64> = FxHashSet::default();
3989        let mut all_sccs: FxHashSet<u64> = FxHashSet::default();
3990        for (&vertex, &scc) in &self.retained_scc_members {
3991            all_sccs.insert(scc);
3992            if self.graph.is_dirty(vertex) {
3993                dirty_sccs.insert(scc);
3994                self.retained_scc_dirty_at_begin.push((vertex, scc));
3995            }
3996        }
3997        let reused_members = self
3998            .retained_scc_members
3999            .values()
4000            .filter(|scc| !dirty_sccs.contains(scc))
4001            .count();
4002        let t = &mut self.last_cycle_telemetry;
4003        t.reused_sccs = all_sccs.len() - dirty_sccs.len();
4004        t.reused_scc_members = reused_members;
4005    }
4006
4007    pub fn virtual_dep_fallback_activations(&self) -> u64 {
4008        self.virtual_dep_fallback_activations
4009    }
4010
4011    #[cfg(test)]
4012    pub(crate) fn lookup_index_flights_built_for_test(&self) -> usize {
4013        self.lookup_index_cache
4014            .flights_built
4015            .load(std::sync::atomic::Ordering::Relaxed)
4016    }
4017
4018    pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
4019        self.lookup_index_cache.report()
4020    }
4021
4022    fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
4023        let start_row = view.start_row();
4024        let end_row = view.end_row();
4025        let start_col = view.start_col();
4026        let end_col = view.end_col();
4027        for row in start_row..=end_row {
4028            let Ok(row_u32) = u32::try_from(row) else {
4029                return true;
4030            };
4031            for col in start_col..=end_col {
4032                let Ok(col_u32) = u32::try_from(col) else {
4033                    return true;
4034                };
4035                let cell_ref = self
4036                    .graph
4037                    .make_cell_ref_internal(sheet_id, row_u32, col_u32);
4038                if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
4039                    && self.graph.is_volatile(vertex_id)
4040                {
4041                    return true;
4042                }
4043            }
4044        }
4045        false
4046    }
4047
4048    fn build_lookup_index_impl(
4049        &self,
4050        view: &RangeView<'_>,
4051        axis: LookupAxis,
4052    ) -> Option<Arc<LookupIndex>> {
4053        let (rows, cols) = view.dims();
4054        if rows == 0 || cols == 0 {
4055            self.lookup_index_cache.note_skipped_tiny();
4056            return None;
4057        }
4058        let len = match axis {
4059            LookupAxis::ColumnInView(col) => {
4060                if col >= cols {
4061                    self.lookup_index_cache.note_skipped_tiny();
4062                    return None;
4063                }
4064                rows
4065            }
4066            LookupAxis::RowInView(row) => {
4067                if row >= rows {
4068                    self.lookup_index_cache.note_skipped_tiny();
4069                    return None;
4070                }
4071                cols
4072            }
4073        };
4074        if len < 64 {
4075            self.lookup_index_cache.note_skipped_tiny();
4076            return None;
4077        }
4078
4079        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
4080        let key = LookupIndexKey {
4081            sheet_id,
4082            start_row: u32::try_from(view.start_row()).ok()?,
4083            start_col: u32::try_from(view.start_col()).ok()?,
4084            end_row: u32::try_from(view.end_row()).ok()?,
4085            end_col: u32::try_from(view.end_col()).ok()?,
4086            axis,
4087            snapshot_id: self.data_snapshot_id(),
4088        };
4089        if let Some(index) = self.lookup_index_cache.get(&key) {
4090            return Some(index);
4091        }
4092        if self
4093            .lookup_index_cache
4094            .would_exceed_cap(estimate_bytes(len, 0))
4095        {
4096            self.lookup_index_cache.note_skipped_cap();
4097            return None;
4098        }
4099        if !self.lookup_index_cache.should_build(key) {
4100            return None;
4101        }
4102        // Parallel members of a lookup family miss together: one builds.
4103        self.lookup_index_cache.single_flight(key, || {
4104            if let Some(index) = self.lookup_index_cache.recheck(&key) {
4105                return Some(index);
4106            }
4107            if self.lookup_index_cache.is_known_volatile(&key) {
4108                self.lookup_index_cache.note_skipped_volatile();
4109                return None;
4110            }
4111            if self.lookup_view_contains_volatile(view, sheet_id) {
4112                self.lookup_index_cache.note_volatile_key(key);
4113                self.lookup_index_cache.note_skipped_volatile();
4114                return None;
4115            }
4116            #[cfg(test)]
4117            self.lookup_index_cache
4118                .flights_built
4119                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4120            match LookupIndex::build(view, axis, self.config.date_system).ok()? {
4121                BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
4122                BuildOutcome::ErrorInLookupAxis => {
4123                    self.lookup_index_cache.note_skipped_error();
4124                    None
4125                }
4126                BuildOutcome::Degenerate => {
4127                    self.lookup_index_cache.note_skipped_tiny();
4128                    None
4129                }
4130            }
4131        })
4132    }
4133
4134    fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
4135        if !self.config.enable_virtual_dep_telemetry {
4136            self.last_virtual_dep_telemetry = VirtualDepTelemetry {
4137                fallback_mode_activations: self.virtual_dep_fallback_activations,
4138                ..VirtualDepTelemetry::default()
4139            };
4140        }
4141    }
4142
4143    fn source_cache_session(&self) -> SourceCacheSession {
4144        self.clear_source_cache();
4145        SourceCacheSession {
4146            cache: self.source_cache.clone(),
4147        }
4148    }
4149
4150    fn resolve_source_scalar_cached(
4151        &self,
4152        name: &str,
4153        version: Option<u64>,
4154    ) -> Result<LiteralValue, ExcelError> {
4155        let key = (name.to_string(), version);
4156        if let Ok(mut g) = self.source_cache.write() {
4157            if let Some(v) = g.scalars.get(&key) {
4158                return Ok(v.clone());
4159            }
4160
4161            let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
4162                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4163                    ExcelError::new(ExcelErrorKind::Ref)
4164                        .with_message(format!("Unresolved source scalar: {name}"))
4165                } else {
4166                    err
4167                }
4168            })?;
4169            g.scalars.insert(key, v.clone());
4170            Ok(v)
4171        } else {
4172            self.resolver.resolve_source_scalar(name).map_err(|err| {
4173                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4174                    ExcelError::new(ExcelErrorKind::Ref)
4175                        .with_message(format!("Unresolved source scalar: {name}"))
4176                } else {
4177                    err
4178                }
4179            })
4180        }
4181    }
4182
4183    fn resolve_source_table_cached(
4184        &self,
4185        name: &str,
4186        version: Option<u64>,
4187    ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
4188        let key = (name.to_string(), version);
4189        if let Ok(mut g) = self.source_cache.write() {
4190            if let Some(t) = g.tables.get(&key) {
4191                return Ok(t.clone());
4192            }
4193
4194            let t = self.resolver.resolve_source_table(name).map_err(|err| {
4195                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4196                    ExcelError::new(ExcelErrorKind::Ref)
4197                        .with_message(format!("Unresolved source table: {name}"))
4198                } else {
4199                    err
4200                }
4201            })?;
4202            let t: Arc<dyn crate::traits::Table> = Arc::from(t);
4203            g.tables.insert(key, t.clone());
4204            Ok(t)
4205        } else {
4206            self.resolver
4207                .resolve_source_table(name)
4208                .map_err(|err| {
4209                    if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4210                        ExcelError::new(ExcelErrorKind::Ref)
4211                            .with_message(format!("Unresolved source table: {name}"))
4212                    } else {
4213                        err
4214                    }
4215                })
4216                .map(Arc::from)
4217        }
4218    }
4219
4220    fn source_table_to_range_view(
4221        &self,
4222        table: &dyn crate::traits::Table,
4223        spec: &Option<formualizer_parse::parser::TableSpecifier>,
4224    ) -> Result<RangeView<'static>, ExcelError> {
4225        use formualizer_parse::parser::{SpecialItem, TableSpecifier};
4226
4227        let owned = match spec {
4228            Some(TableSpecifier::Column(c)) => {
4229                let c = c.trim();
4230                if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
4231                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4232                        "Complex structured references not yet supported".to_string(),
4233                    ));
4234                }
4235                table.get_column(c)?.materialise().into_owned()
4236            }
4237            Some(TableSpecifier::ColumnRange(start, end)) => {
4238                let cols = table.columns();
4239                let start = start.trim();
4240                let end = end.trim();
4241                let start_key = start.to_lowercase();
4242                let end_key = end.to_lowercase();
4243                let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
4244                let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
4245                if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
4246                    if si > ei {
4247                        std::mem::swap(&mut si, &mut ei);
4248                    }
4249                    let h = table.data_height();
4250                    let w = ei - si + 1;
4251                    let mut rows = vec![vec![LiteralValue::Empty; w]; h];
4252                    for (offset, ci) in (si..=ei).enumerate() {
4253                        let cname = &cols[ci];
4254                        let col_range = table.get_column(cname)?;
4255                        let (rh, _) = col_range.dimensions();
4256                        for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
4257                            row[offset] = col_range.get(r, 0)?;
4258                        }
4259                    }
4260                    rows
4261                } else {
4262                    return Err(ExcelError::new(ExcelErrorKind::Ref)
4263                        .with_message("Column range refers to unknown column(s)".to_string()));
4264                }
4265            }
4266            Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
4267            | Some(TableSpecifier::Headers) => table
4268                .headers_row()
4269                .map(|r| r.materialise().into_owned())
4270                .unwrap_or_default(),
4271            Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
4272            | Some(TableSpecifier::Totals) => table
4273                .totals_row()
4274                .map(|r| r.materialise().into_owned())
4275                .unwrap_or_default(),
4276            Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
4277                table
4278                    .data_body()
4279                    .map(|r| r.materialise().into_owned())
4280                    .unwrap_or_default()
4281            }
4282            Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
4283                let mut out: Vec<Vec<LiteralValue>> = Vec::new();
4284                if let Some(h) = table.headers_row() {
4285                    out.extend(h.iter_rows());
4286                }
4287                if let Some(body) = table.data_body() {
4288                    out.extend(body.iter_rows());
4289                }
4290                if let Some(tr) = table.totals_row() {
4291                    out.extend(tr.iter_rows());
4292                }
4293                out
4294            }
4295            Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
4296                return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4297                    "@ (This Row) requires table-aware context; not yet supported".to_string(),
4298                ));
4299            }
4300            Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
4301                return Err(ExcelError::new(ExcelErrorKind::NImpl)
4302                    .with_message("Complex structured references not yet supported".to_string()));
4303            }
4304            None => {
4305                return Err(ExcelError::new(ExcelErrorKind::NImpl)
4306                    .with_message("Table reference without specifier is unsupported".to_string()));
4307            }
4308        };
4309
4310        Ok(RangeView::from_owned_rows(owned, self.config.date_system))
4311    }
4312
4313    pub fn default_sheet_id(&self) -> SheetId {
4314        self.graph.default_sheet_id()
4315    }
4316
4317    pub fn default_sheet_name(&self) -> &str {
4318        self.graph.default_sheet_name()
4319    }
4320
4321    /// Update the workbook seed for deterministic RNGs in functions.
4322    pub fn set_workbook_seed(&mut self, seed: u64) {
4323        self.config.workbook_seed = seed;
4324    }
4325
4326    /// Set the volatile level policy (Always/OnRecalc/OnOpen)
4327    pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
4328        self.config.volatile_level = level;
4329    }
4330
4331    /// Set public temporal materialisation to native values or raw serials.
4332    pub fn set_temporal_egress(&mut self, policy: crate::engine::TemporalEgress) {
4333        self.config.temporal_egress = policy;
4334    }
4335
4336    pub fn temporal_egress(&self) -> crate::engine::TemporalEgress {
4337        self.config.temporal_egress
4338    }
4339
4340    /// Enable/disable deterministic evaluation mode (fixed clock + timezone).
4341    pub fn set_deterministic_mode(
4342        &mut self,
4343        mode: crate::engine::DeterministicMode,
4344    ) -> Result<(), ExcelError> {
4345        let clock = mode.build_clock()?;
4346        self.config.deterministic_mode = mode;
4347        self.clock = crate::timezone::SnapshotClock::new(clock);
4348        Ok(())
4349    }
4350
4351    /// Inject a custom [`ClockProvider`](crate::timezone::ClockProvider) for
4352    /// volatile date/time builtins (`NOW()`, `TODAY()`).
4353    ///
4354    /// The provider is the clock *source*; per spec §7.11 the engine samples
4355    /// it once at the start of every evaluation request and all reads within
4356    /// that recalc (including SCC iteration passes) observe the frozen
4357    /// sample.
4358    pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
4359        self.clock = crate::timezone::SnapshotClock::new(clock);
4360    }
4361
4362    fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
4363        self.config.deterministic_mode.validate()
4364    }
4365
4366    pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
4367        self.graph.sheet_id(name)
4368    }
4369
4370    pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
4371        self.add_sheet(name)
4372            .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
4373    }
4374
4375    pub fn sheet_name(&self, id: SheetId) -> &str {
4376        self.graph.sheet_name(id)
4377    }
4378
4379    fn arrow_sheet_position(&self, id: SheetId) -> Option<usize> {
4380        let index = self.arrow_sheet_index.get_or_init(|| {
4381            let mut index = Vec::new();
4382            for (position, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
4383                if let Some(id) = self.graph.sheet_id(&sheet.name) {
4384                    let needed = usize::from(id).saturating_add(1);
4385                    if index.len() < needed {
4386                        index.resize(needed, None);
4387                    }
4388                    if let Some(slot) = index.get_mut(usize::from(id)) {
4389                        *slot = Some(position);
4390                    }
4391                }
4392            }
4393            index
4394        });
4395        index.get(usize::from(id)).copied().flatten()
4396    }
4397
4398    /// Indexed Arrow storage lookup by stable registry identity.
4399    pub(crate) fn arrow_sheet_by_id(&self, id: SheetId) -> Option<&crate::arrow_store::ArrowSheet> {
4400        self.arrow_sheets.sheets.get(self.arrow_sheet_position(id)?)
4401    }
4402
4403    /// Mutable counterpart of [`Self::arrow_sheet_by_id`].
4404    pub(crate) fn arrow_sheet_by_id_mut(
4405        &mut self,
4406        id: SheetId,
4407    ) -> Option<&mut crate::arrow_store::ArrowSheet> {
4408        let position = self.arrow_sheet_position(id)?;
4409        self.arrow_sheets.sheets.get_mut(position)
4410    }
4411
4412    fn rebuild_arrow_sheet_index(&mut self) {
4413        self.arrow_sheet_index.take();
4414        self.resolved_arrow_sheets.clear();
4415        for (index, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
4416            if let Some(id) = self.graph.sheet_id(&sheet.name) {
4417                let len = usize::from(id).saturating_add(1);
4418                if self.resolved_arrow_sheets.len() < len {
4419                    self.resolved_arrow_sheets.resize(len, None);
4420                }
4421                if let Some(slot) = self.resolved_arrow_sheets.get_mut(usize::from(id)) {
4422                    *slot = Some(index);
4423                }
4424            }
4425        }
4426    }
4427
4428    pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
4429        let id = self.graph.add_sheet(name)?;
4430        self.ensure_arrow_sheet(name);
4431        self.mark_topology_edited();
4432        Ok(id)
4433    }
4434
4435    pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
4436        let source_id = self.graph.sheet_id(source).ok_or_else(|| {
4437            ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
4438        })?;
4439        if new_name.is_empty() || new_name.len() > 255 {
4440            return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
4441        }
4442        if self.graph.sheet_id(new_name).is_some() {
4443            return Err(ExcelError::new(ExcelErrorKind::Value)
4444                .with_message(format!("Sheet '{new_name}' already exists")));
4445        }
4446        let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
4447
4448        if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
4449            let mut copied_sheet = source_sheet;
4450            copied_sheet.name = Arc::<str>::from(new_name);
4451            self.arrow_sheets.sheets.push(copied_sheet);
4452            self.rebuild_arrow_sheet_index();
4453        } else {
4454            self.ensure_arrow_sheet(new_name);
4455        }
4456
4457        let duplicated_formulas = self
4458            .graph
4459            .formula_vertices()
4460            .into_iter()
4461            .filter(|vertex| {
4462                self.graph
4463                    .get_cell_ref(*vertex)
4464                    .is_some_and(|cell| cell.sheet_id == new_id)
4465            })
4466            .collect::<Vec<_>>();
4467        self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
4468        self.mark_topology_edited();
4469        Ok(new_id)
4470    }
4471
4472    fn ensure_arrow_sheet(&mut self, name: &str) {
4473        if self.arrow_sheets.sheet(name).is_some() {
4474            return;
4475        }
4476        self.arrow_sheets
4477            .sheets
4478            .push(crate::arrow_store::ArrowSheet {
4479                name: std::sync::Arc::<str>::from(name),
4480                date_system: self.config.date_system,
4481                columns: Vec::new(),
4482                nrows: 0,
4483                chunk_starts: Vec::new(),
4484                chunk_rows: 32 * 1024,
4485            });
4486        self.rebuild_arrow_sheet_index();
4487    }
4488
4489    pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
4490        let name = self.graph.sheet_name(sheet_id).to_string();
4491        self.purge_derived_formats_for_sheet(sheet_id);
4492        self.graph.remove_sheet(sheet_id)?;
4493        self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
4494        self.rebuild_arrow_sheet_index();
4495        // Sheet removal can change cross-sheet refs, names, and default-sheet
4496        // resolution. Until those domains have a complete exact dependency
4497        // proof, retain the documented graph-owned global invalidation.
4498        self.clear_all_computed_overlays();
4499        self.mark_all_formula_vertices_dirty();
4500        self.clear_staged_formulas_for_sheet(&name);
4501        if self.row_visibility.remove(&sheet_id).is_some() {
4502            self.invalidate_row_visibility_mask_cache();
4503        }
4504        self.record_structural_change(StructuralScope::RemovedSheet(sheet_id));
4505        self.mark_topology_edited();
4506        Ok(())
4507    }
4508
4509    /// Helper to synchronize the Arrow-backed storage layer.
4510    fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
4511        if let Some(asheet) = self
4512            .arrow_sheets
4513            .sheets
4514            .iter_mut()
4515            .find(|s| s.name.as_ref() == target_name)
4516        {
4517            asheet.name = std::sync::Arc::<str>::from(new_name);
4518            return true;
4519        }
4520        false
4521    }
4522
4523    pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4524        let old_name = self.graph.sheet_name(sheet_id).to_string();
4525
4526        // Speculative Storage Update
4527        // Update name in storage FIRST so the Evaluator can find it during Graph rescue.
4528        self.rename_sheet_in_arrow_store(&old_name, new_name);
4529
4530        // Graph Update (Metadata + Rescue Logic)
4531        match self.graph.rename_sheet(sheet_id, new_name) {
4532            Ok(_) => {
4533                self.rename_staged_formula_sheet(&old_name, new_name);
4534                // Success! Invalidate cache for the moved sheet
4535                let sheet_vertices: Vec<VertexId> = self
4536                    .graph
4537                    .grid_vertices_in_sheet(sheet_id)
4538                    .map(|(id, _)| id)
4539                    .collect();
4540                for v_id in sheet_vertices {
4541                    self.graph.mark_vertex_dirty(v_id);
4542                }
4543                // Sheet rename preserves SheetId and therefore formula dependencies.
4544                self.mark_topology_edited();
4545                Ok(())
4546            }
4547            Err(e) => {
4548                // ROLLBACK: Revert storage if graph rejected the name
4549                self.rename_sheet_in_arrow_store(new_name, &old_name);
4550                Err(e)
4551            }
4552        }
4553    }
4554
4555    pub fn named_ranges_iter(
4556        &self,
4557    ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4558        self.graph.named_ranges_iter()
4559    }
4560
4561    pub fn sheet_named_ranges_iter(
4562        &self,
4563    ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4564        self.graph.sheet_named_ranges_iter()
4565    }
4566
4567    pub fn resolve_name_entry(
4568        &self,
4569        name: &str,
4570        current_sheet: SheetId,
4571    ) -> Option<&crate::engine::named_range::NamedRange> {
4572        self.graph.resolve_name_entry(name, current_sheet)
4573    }
4574
4575    /// The [`NameScope`] an optional scope-sheet argument denotes.
4576    ///
4577    /// `None` means **workbook scope**, not "the default sheet": a caller that
4578    /// supplies no sheet context is asking about workbook-scoped names only.
4579    /// An unknown sheet name is a malformed query and errors rather than
4580    /// silently degrading to another sheet's scope (issue #110).
4581    ///
4582    /// This is the one owned derivation from `Option<&str>` to a name scope;
4583    /// every scope-taking entry point routes through it.
4584    pub(crate) fn name_query_scope(
4585        &self,
4586        scope_sheet: Option<&str>,
4587    ) -> Result<NameScope, ExcelError> {
4588        match scope_sheet {
4589            None => Ok(NameScope::Workbook),
4590            Some(sheet) => self
4591                .graph
4592                .sheet_id(sheet)
4593                .map(NameScope::Sheet)
4594                .ok_or_else(|| {
4595                    ExcelError::new(ExcelErrorKind::Ref)
4596                        .with_message(format!("name scope sheet not found: {sheet}"))
4597                }),
4598        }
4599    }
4600
4601    /// Whether `name` resolves in the scope denoted by `scope_sheet`.
4602    ///
4603    /// `scope_sheet == None` asks about workbook scope only; a name scoped to a
4604    /// single sheet (including the default sheet) does not answer it. An unknown
4605    /// sheet name resolves nothing.
4606    /// Resolve a [`SharedSheetLocator`](crate::reference::SharedSheetLocator)
4607    /// against an explicit context sheet.
4608    ///
4609    /// Thin forwarder to
4610    /// [`SheetRegistry::resolve_locator`](crate::engine::sheet_registry::SheetRegistry::resolve_locator),
4611    /// the single owned derivation. `Current` resolves to `context_sheet`, never
4612    /// to the workbook's default sheet.
4613    fn resolve_sheet_locator(
4614        &self,
4615        locator: &crate::reference::SharedSheetLocator<'_>,
4616        context_sheet: SheetId,
4617    ) -> Result<SheetId, ExcelError> {
4618        self.graph
4619            .sheet_reg()
4620            .resolve_locator(locator, context_sheet)
4621    }
4622
4623    pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4624        let Ok(scope) = self.name_query_scope(scope_sheet) else {
4625            return false;
4626        };
4627        self.graph
4628            .resolve_name_entry_in_scope(name, scope)
4629            .is_some()
4630    }
4631
4632    /// The current value of `name` in the scope denoted by `scope_sheet`.
4633    ///
4634    /// Scoping follows [`Self::has_name`]: `None` is workbook scope only.
4635    pub fn resolved_name_value(
4636        &self,
4637        name: &str,
4638        scope_sheet: Option<&str>,
4639    ) -> Option<LiteralValue> {
4640        let scope = self.name_query_scope(scope_sheet).ok()?;
4641        let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4642        self.graph.get_value(entry.vertex)
4643    }
4644
4645    pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4646        let entry = self.graph.resolve_table_entry(name)?;
4647        Some(TableMetadata {
4648            name: entry.name.clone(),
4649            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4650            start_row: entry.range.start.coord.row() + 1,
4651            start_col: entry.range.start.coord.col() + 1,
4652            end_row: entry.range.end.coord.row() + 1,
4653            end_col: entry.range.end.coord.col() + 1,
4654            header_row: entry.header_row,
4655            headers: entry.headers.clone(),
4656            totals_row: entry.totals_row,
4657        })
4658    }
4659
4660    /// Metadata for every defined table, ordered by name.
4661    pub fn tables(&self) -> Vec<TableMetadata> {
4662        self.graph
4663            .table_names()
4664            .into_iter()
4665            .filter_map(|name| self.table_metadata(&name))
4666            .collect()
4667    }
4668
4669    pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4670        let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4671
4672        for (name, named) in self.graph.named_ranges_iter() {
4673            out.push(crate::engine::named_range::NamedRangeSnapshot {
4674                name: name.clone(),
4675                scope: NameScope::Workbook,
4676                definition: named.definition.clone(),
4677            });
4678        }
4679
4680        for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4681            out.push(crate::engine::named_range::NamedRangeSnapshot {
4682                name: name.clone(),
4683                scope: NameScope::Sheet(*sheet_id),
4684                definition: named.definition.clone(),
4685            });
4686        }
4687
4688        out.sort_by(|a, b| {
4689            let a_scope = match a.scope {
4690                NameScope::Workbook => (0u8, 0u32),
4691                NameScope::Sheet(id) => (1u8, u32::from(id)),
4692            };
4693            let b_scope = match b.scope {
4694                NameScope::Workbook => (0u8, 0u32),
4695                NameScope::Sheet(id) => (1u8, u32::from(id)),
4696            };
4697            a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4698        });
4699
4700        out
4701    }
4702
4703    pub fn named_ranges_snapshot_for_sheet(
4704        &self,
4705        sheet_id: SheetId,
4706    ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4707        self.named_ranges_snapshot()
4708            .into_iter()
4709            .filter(|entry| match entry.scope {
4710                NameScope::Workbook => true,
4711                NameScope::Sheet(id) => id == sheet_id,
4712            })
4713            .collect()
4714    }
4715
4716    pub fn define_name(
4717        &mut self,
4718        name: &str,
4719        definition: NamedDefinition,
4720        scope: NameScope,
4721    ) -> Result<(), ExcelError> {
4722        self.graph.validate_define_name(name, scope)?;
4723        self.graph.define_name(name, definition, scope)?;
4724        self.record_structural_change(StructuralScope::AllSheets);
4725
4726        self.mark_topology_edited();
4727
4728        Ok(())
4729    }
4730
4731    pub fn update_name(
4732        &mut self,
4733        name: &str,
4734        definition: NamedDefinition,
4735        scope: NameScope,
4736    ) -> Result<(), ExcelError> {
4737        self.graph.validate_existing_name(name, scope)?;
4738        self.graph.update_name(name, definition, scope)?;
4739        self.record_structural_change(StructuralScope::AllSheets);
4740
4741        self.mark_topology_edited();
4742
4743        Ok(())
4744    }
4745
4746    pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4747        self.graph.validate_existing_name(name, scope)?;
4748        self.graph.delete_name(name, scope)?;
4749        self.record_structural_change(StructuralScope::AllSheets);
4750
4751        self.mark_topology_edited();
4752
4753        Ok(())
4754    }
4755
4756    pub fn define_table(
4757        &mut self,
4758        name: &str,
4759        range: crate::reference::RangeRef,
4760        header_row: bool,
4761        headers: Vec<String>,
4762        totals_row: bool,
4763    ) -> Result<(), ExcelError> {
4764        self.graph
4765            .define_table(name, range, header_row, headers, totals_row)?;
4766        self.record_structural_change(StructuralScope::AllSheets);
4767        self.mark_topology_edited();
4768        Ok(())
4769    }
4770
4771    pub fn define_source_scalar(
4772        &mut self,
4773        name: &str,
4774        version: Option<u64>,
4775    ) -> Result<(), ExcelError> {
4776        self.graph.define_source_scalar(name, version)?;
4777        self.record_structural_change(StructuralScope::OpaqueGlobal);
4778        self.mark_topology_edited();
4779        Ok(())
4780    }
4781
4782    pub fn define_source_table(
4783        &mut self,
4784        name: &str,
4785        version: Option<u64>,
4786    ) -> Result<(), ExcelError> {
4787        self.graph.define_source_table(name, version)?;
4788        self.record_structural_change(StructuralScope::OpaqueGlobal);
4789        self.mark_topology_edited();
4790        Ok(())
4791    }
4792
4793    pub fn set_source_scalar_version(
4794        &mut self,
4795        name: &str,
4796        version: Option<u64>,
4797    ) -> Result<(), ExcelError> {
4798        self.graph.set_source_scalar_version(name, version)?;
4799        Ok(())
4800    }
4801
4802    pub fn set_source_table_version(
4803        &mut self,
4804        name: &str,
4805        version: Option<u64>,
4806    ) -> Result<(), ExcelError> {
4807        self.graph.set_source_table_version(name, version)?;
4808        Ok(())
4809    }
4810
4811    pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4812        self.graph.invalidate_source(name)?;
4813        Ok(())
4814    }
4815
4816    pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4817        self.graph.get_value(vertex)
4818    }
4819
4820    pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4821        self.graph.get_cell_value(sheet, row, col)
4822    }
4823
4824    pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4825        self.graph.get_vertex_for_cell(cell)
4826    }
4827
4828    pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4829        self.graph.get_evaluation_vertices()
4830    }
4831
4832    /// Return read-only baseline counters for dispatch benchmarking.
4833    pub fn baseline_stats(&self) -> EngineBaselineStats {
4834        let graph = self.graph.baseline_stats();
4835        EngineBaselineStats {
4836            graph_vertex_count: graph.graph_vertex_count,
4837            graph_formula_vertex_count: graph.graph_formula_vertex_count,
4838            graph_edge_count: graph.graph_edge_count,
4839            dirty_vertex_count: graph.dirty_vertex_count,
4840            evaluation_vertex_count: graph.evaluation_vertex_count,
4841            formula_ast_root_count: graph.formula_ast_root_count,
4842            formula_ast_node_count: graph.formula_ast_node_count,
4843            staged_formula_count: self.staged_formula_count(),
4844            formula_plane_active_span_count: 0,
4845            formula_plane_producer_result_entries: 0,
4846            formula_plane_consumer_read_entries: 0,
4847            formula_plane_mixed_topology_cache_builds: 0,
4848            formula_plane_mixed_topology_cache_hits: 0,
4849            formula_plane_mixed_topology_cache_overflows: 0,
4850            formula_plane_dirty_pending_events: 0,
4851            formula_plane_dirty_region_events_recorded: 0,
4852            formula_plane_dirty_span_region_events_recorded: 0,
4853            formula_plane_dirty_whole_span_seeds_recorded: 0,
4854            formula_plane_dirty_global_invalidations: 0,
4855            formula_plane_structural_span_candidates: 0,
4856            formula_plane_cycle_member_span_demotions: 0,
4857            formula_plane_array_result_span_demotions: 0,
4858            retained_scc_members: self.retained_scc_members.len(),
4859        }
4860    }
4861
4862    /// Mutation revision captured by read-only engine reports.
4863    pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4864        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4865    }
4866
4867    #[cfg(test)]
4868    pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4869        self.used_axis_bounds_cache
4870            .read()
4871            .ok()
4872            .and_then(|guard| {
4873                guard.as_ref().map(|cache| {
4874                    (
4875                        cache.row_hits.load(Ordering::Relaxed),
4876                        cache.row_misses.load(Ordering::Relaxed),
4877                        cache.col_hits.load(Ordering::Relaxed),
4878                        cache.col_misses.load(Ordering::Relaxed),
4879                    )
4880                })
4881            })
4882            .unwrap_or((0, 0, 0, 0))
4883    }
4884
4885    pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4886        self.graph.set_first_load_assume_new(enabled);
4887    }
4888
4889    pub fn first_load_assume_new(&self) -> bool {
4890        self.graph.first_load_assume_new()
4891    }
4892
4893    pub fn reset_ensure_touched(&mut self) {
4894        self.graph.reset_ensure_touched();
4895    }
4896
4897    pub fn finalize_sheet_index(&mut self, sheet: &str) {
4898        self.graph.finalize_sheet_index(sheet);
4899    }
4900
4901    /// Execute a named Engine action.
4902    ///
4903    /// Ticket 614 introduces this as the stable Engine-level transaction surface.
4904    /// For now actions are commit-only: they do not create changelog boundaries and they do not
4905    /// provide rollback/atomicity.
4906    ///
4907    /// Nested actions are deterministically handled by *disallowing* nesting: calling
4908    /// `Engine::action` while another action is active returns `EditorError::TransactionFailed`.
4909    pub fn action<T>(
4910        &mut self,
4911        name: impl AsRef<str>,
4912        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4913    ) -> Result<T, crate::engine::EditorError> {
4914        if self.action_depth != 0 {
4915            return Err(crate::engine::EditorError::TransactionFailed {
4916                reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4917                    .to_string(),
4918            });
4919        }
4920
4921        self.action_depth = 1;
4922        let engine_ptr: *mut Engine<R> = self;
4923        let _guard = ActionDepthGuard {
4924            engine: engine_ptr,
4925            _marker: std::marker::PhantomData,
4926        };
4927
4928        let mut tx = EngineAction {
4929            engine: self,
4930            name: name.as_ref().to_string(),
4931            capture: None,
4932            arrow_undo: None,
4933            atomic_policy: false,
4934        };
4935        f(&mut tx)
4936    }
4937
4938    /// Execute a named Engine action with atomic commit/rollback semantics.
4939    ///
4940    /// This variant does not require a `ChangeLog` and uses an internal journal for rollback.
4941    pub fn action_atomic<T>(
4942        &mut self,
4943        name: impl Into<String>,
4944        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4945    ) -> Result<T, crate::engine::EditorError> {
4946        let (v, _j) = self.action_atomic_journal(name, f)?;
4947        Ok(v)
4948    }
4949
4950    /// Like `action_atomic`, but returns the committed journal entry for undo/redo storage.
4951    pub fn action_atomic_journal<T>(
4952        &mut self,
4953        name: impl Into<String>,
4954        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4955    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4956        if self.action_depth != 0 {
4957            return Err(crate::engine::EditorError::TransactionFailed {
4958                reason: "Nested Engine::action calls are not supported (deterministic rule)"
4959                    .to_string(),
4960            });
4961        }
4962
4963        self.action_depth = 1;
4964        let engine_ptr: *mut Engine<R> = self;
4965        let _guard = ActionDepthGuard {
4966            engine: engine_ptr,
4967            _marker: std::marker::PhantomData,
4968        };
4969
4970        let name_str = name.into();
4971        let mut capture = MutationCapture::new(Default::default());
4972        let start_len = capture.len();
4973        self.action_atomic_impl(&mut capture, start_len, true, name_str, f)
4974    }
4975
4976    fn action_atomic_impl<T>(
4977        &mut self,
4978        capture: &mut MutationCapture,
4979        start_len: usize,
4980        expand_runs: bool,
4981        name: String,
4982        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4983    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4984        let invalidation_baseline = self.invalidation_baseline();
4985        let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4986        let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4987
4988        let capture_ptr: *mut MutationCapture = capture;
4989        let mut tx = EngineAction {
4990            engine: self,
4991            name: name.clone(),
4992            capture: Some(capture_ptr),
4993            arrow_undo: Some(arrow_ptr),
4994            atomic_policy: true,
4995        };
4996
4997        let res = f(&mut tx);
4998
4999        // Capture graph structural delta for this action. The journal is a
5000        // public value: run records (Program 2) are expanded into it, except
5001        // for a caller that only publishes the capture to its change log
5002        // (`expand_runs` false), where the journal (plain events) drives
5003        // invalidation and records count as topology changes; a rollback
5004        // still replays the expanded events.
5005        let capture_ref = unsafe { &*capture_ptr };
5006        let has_runs = capture_ref.lazy_len() > 0;
5007        let graph_events: Vec<crate::engine::ChangeEvent> = if expand_runs || res.is_err() {
5008            capture_ref.expanded_events_from(start_len, 0)
5009        } else {
5010            capture_ref.events()[start_len..].to_vec()
5011        };
5012        let graph_batch = crate::engine::GraphUndoBatch {
5013            events: graph_events,
5014        };
5015        let affected_cells = arrow_undo.ops.len();
5016        let journal = crate::engine::ActionJournal {
5017            name,
5018            graph: graph_batch,
5019            arrow: arrow_undo,
5020            affected_cells,
5021        };
5022
5023        match res {
5024            Ok(v) => {
5025                if !journal.graph.is_empty() || !journal.arrow.is_empty() || has_runs {
5026                    for event in &journal.graph.events {
5027                        self.record_change_for_event(event);
5028                    }
5029                    let mut impact = Self::classify_change_events(
5030                        &journal.graph.events,
5031                        LoggedEditDirection::Original,
5032                    )
5033                    .max(Self::classify_arrow_undo(&journal.arrow));
5034                    if has_runs {
5035                        impact = impact.max(LoggedEditImpact::Topology);
5036                    }
5037                    self.apply_logged_edit_impact(impact, invalidation_baseline);
5038                }
5039                Ok((v, journal))
5040            }
5041            Err(e) => {
5042                if let Err(rb) = self.rollback_from_action_journal(&journal, invalidation_baseline)
5043                {
5044                    return Err(crate::engine::EditorError::TransactionFailed {
5045                        reason: format!(
5046                            "Engine::action_atomic rollback failed after error '{e}': {rb}"
5047                        ),
5048                    });
5049                }
5050                if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5051                    for event in &journal.graph.events {
5052                        self.record_change_for_event(event);
5053                    }
5054                }
5055                Err(e)
5056            }
5057        }
5058    }
5059
5060    /// Execute a named Engine action, logging graph changes into the provided ChangeLog.
5061    ///
5062    /// Ticket 615: this variant provides atomicity. If the action returns an error, it rolls back:
5063    /// - Dependency graph structural edits (via inverse ChangeEvents)
5064    /// - Arrow-truth overlay writes mirrored from ChangeEvents
5065    /// - ChangeLog entries (published only after a successful commit)
5066    pub fn action_with_logger<T>(
5067        &mut self,
5068        log: &mut crate::engine::ChangeLog,
5069        name: impl AsRef<str>,
5070        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5071    ) -> Result<T, crate::engine::EditorError> {
5072        if self.action_depth != 0 {
5073            return Err(crate::engine::EditorError::TransactionFailed {
5074                reason: "Nested Engine::action calls are not supported (deterministic rule)"
5075                    .to_string(),
5076            });
5077        }
5078
5079        self.action_depth = 1;
5080        let engine_ptr: *mut Engine<R> = self;
5081        let _guard = ActionDepthGuard {
5082            engine: engine_ptr,
5083            _marker: std::marker::PhantomData,
5084        };
5085
5086        let name_str = name.as_ref().to_string();
5087        let mut capture = MutationCapture::new(log.current_meta());
5088        let start_len = capture.len();
5089        capture.begin_compound(name_str.clone());
5090
5091        // Mutation correctness uses the complete private capture. The provided ChangeLog remains
5092        // an observability sink and is not touched until the action outcome is known.
5093        let res = self.action_atomic_impl(&mut capture, start_len, false, name_str, f);
5094        capture.close_compounds();
5095
5096        match res {
5097            Ok((v, _journal)) => {
5098                log.publish_capture(capture);
5099                Ok(v)
5100            }
5101            Err(e) => {
5102                // Preserve sequence/group gaps without retaining failed events or evicting history.
5103                log.discard_capture(capture);
5104                Err(e)
5105            }
5106        }
5107    }
5108
5109    fn rollback_from_action_journal(
5110        &mut self,
5111        journal: &crate::engine::ActionJournal,
5112        invalidation_baseline: InvalidationBaseline,
5113    ) -> Result<(), crate::engine::EditorError> {
5114        // Invalidate first so a partial inverse failure cannot leave a changed
5115        // graph behind an apparently current schedule or lookup cache.
5116        self.invalidate_for_action_journal(
5117            journal,
5118            LoggedEditDirection::InverseReplay,
5119            invalidation_baseline,
5120        );
5121        // 1) Roll back the dependency graph structure.
5122        journal.graph.undo(&mut self.graph)?;
5123        // 2) Roll back engine row-visibility sidecar events.
5124        self.apply_inverse_row_visibility_events(&journal.graph.events);
5125        // 3) Roll back Arrow-truth overlays.
5126        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
5127        Ok(())
5128    }
5129
5130    fn rollback_from_change_events(
5131        &mut self,
5132        events: &[crate::engine::ChangeEvent],
5133        invalidation_baseline: InvalidationBaseline,
5134    ) -> Result<(), crate::engine::EditorError> {
5135        use crate::engine::ChangeEvent;
5136
5137        // Fail closed before applying inverses because replay can return after
5138        // only part of the batch has been restored.
5139        self.invalidate_for_change_events(
5140            events,
5141            LoggedEditDirection::InverseReplay,
5142            invalidation_baseline,
5143        );
5144
5145        // 1) Roll back the dependency graph.
5146        self.graph
5147            .authority_set_replay(crate::engine::authority::history::Replay::Undo);
5148        let rolled_back = (|| {
5149            let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
5150            let mut compound_stack: Vec<usize> = Vec::new();
5151            for (i, ev) in events.iter().enumerate().rev() {
5152                match ev {
5153                    ChangeEvent::CompoundEnd { depth } => {
5154                        compound_stack.push(*depth);
5155                        if let Some(description) =
5156                            crate::engine::graph::editor::change_log::compound_start_description(
5157                                i,
5158                                |j| &events[j],
5159                            )
5160                        {
5161                            editor.inverse_compound_end(description);
5162                        }
5163                    }
5164                    ChangeEvent::CompoundStart { depth, .. } => {
5165                        if compound_stack.last() == Some(depth) {
5166                            compound_stack.pop();
5167                        }
5168                        editor.apply_inverse(ev.clone())?;
5169                    }
5170                    ChangeEvent::SetRowVisibility { .. } => {
5171                        // Engine-side metadata handled after dropping graph editor borrow.
5172                    }
5173                    _ => {
5174                        editor.apply_inverse(ev.clone())?;
5175                    }
5176                }
5177            }
5178            Ok::<_, crate::engine::EditorError>(())
5179        })();
5180        self.graph
5181            .authority_set_replay(crate::engine::authority::history::Replay::Forward);
5182        rolled_back?;
5183
5184        // 2) Roll back engine row-visibility metadata.
5185        for ev in events.iter().rev() {
5186            self.apply_inverse_row_visibility_event(ev);
5187        }
5188
5189        // 3) Roll back Arrow-truth overlays mirrored from those ChangeEvents.
5190        for ev in events.iter().rev() {
5191            self.mirror_inverse_change_to_arrow(ev);
5192        }
5193
5194        Ok(())
5195    }
5196
5197    fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
5198        let sheet_id = self.graph.sheet_id(sheet)?;
5199        let coord = Coord::from_excel(row, col, true, true);
5200        let cell = CellRef::new(sheet_id, coord);
5201        let vid = self.graph.get_vertex_for_cell(&cell)?;
5202        self.graph.get_formula(vid)
5203    }
5204
5205    pub fn define_name_with_logger(
5206        &mut self,
5207        log: &mut crate::engine::ChangeLog,
5208        name: &str,
5209        definition: NamedDefinition,
5210        scope: NameScope,
5211    ) -> Result<(), crate::engine::EditorError> {
5212        self.graph
5213            .validate_define_name(name, scope)
5214            .map_err(crate::engine::EditorError::Excel)?;
5215
5216        {
5217            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5218            editor.define_name(name, definition, scope)?;
5219        }
5220        self.record_structural_change(StructuralScope::AllSheets);
5221
5222        self.mark_topology_edited();
5223
5224        Ok(())
5225    }
5226
5227    pub fn update_name_with_logger(
5228        &mut self,
5229        log: &mut crate::engine::ChangeLog,
5230        name: &str,
5231        definition: NamedDefinition,
5232        scope: NameScope,
5233    ) -> Result<(), crate::engine::EditorError> {
5234        self.graph
5235            .validate_existing_name(name, scope)
5236            .map_err(crate::engine::EditorError::Excel)?;
5237        {
5238            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5239            editor.update_name(name, definition, scope)?;
5240        }
5241        self.record_structural_change(StructuralScope::AllSheets);
5242
5243        self.mark_topology_edited();
5244
5245        Ok(())
5246    }
5247
5248    pub fn delete_name_with_logger(
5249        &mut self,
5250        log: &mut crate::engine::ChangeLog,
5251        name: &str,
5252        scope: NameScope,
5253    ) -> Result<(), crate::engine::EditorError> {
5254        self.graph
5255            .validate_existing_name(name, scope)
5256            .map_err(crate::engine::EditorError::Excel)?;
5257        {
5258            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5259            editor.delete_name(name, scope)?;
5260        }
5261        self.record_structural_change(StructuralScope::AllSheets);
5262
5263        self.mark_topology_edited();
5264
5265        Ok(())
5266    }
5267
5268    pub fn edit_with_logger<T>(
5269        &mut self,
5270        log: &mut crate::engine::ChangeLog,
5271        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5272    ) -> Result<T, crate::engine::EditorError> {
5273        let mut capture = MutationCapture::new(log.current_meta());
5274        let result = self.edit_with_capture(&mut capture, f);
5275        capture.close_compounds();
5276        match result {
5277            Ok(value) => {
5278                log.publish_capture(capture);
5279                Ok(value)
5280            }
5281            Err(error) => {
5282                log.discard_capture(capture);
5283                Err(error)
5284            }
5285        }
5286    }
5287
5288    fn edit_with_capture<T>(
5289        &mut self,
5290        capture: &mut MutationCapture,
5291        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5292    ) -> Result<T, crate::engine::EditorError> {
5293        let invalidation_baseline = self.invalidation_baseline();
5294        let start_len = capture.len();
5295        let lazy_start = capture.lazy_len();
5296
5297        // Provide a spill snapshot reader so VertexEditor can snapshot Arrow-truth spill values
5298        // (graph value cache is intentionally empty in canonical mode).
5299        struct ArrowSpillReader<'a> {
5300            sheets: &'a crate::arrow_store::SheetStore,
5301        }
5302        impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
5303            fn read_cell_value(
5304                &self,
5305                sheet: &str,
5306                row: u32,
5307                col: u32,
5308            ) -> Option<formualizer_common::LiteralValue> {
5309                use formualizer_common::LiteralValue;
5310                let asheet = self.sheets.sheet(sheet)?;
5311                let r0 = row.saturating_sub(1) as usize;
5312                let c0 = col.saturating_sub(1) as usize;
5313                let v = asheet.get_cell_value(r0, c0);
5314                if matches!(v, LiteralValue::Empty) {
5315                    None
5316                } else {
5317                    Some(v)
5318                }
5319            }
5320        }
5321
5322        let ret = {
5323            let spill_reader = ArrowSpillReader {
5324                sheets: &self.arrow_sheets,
5325            };
5326            let mut editor = crate::engine::VertexEditor::with_capture_and_spill_reader(
5327                &mut self.graph,
5328                capture,
5329                &spill_reader,
5330            );
5331            f(&mut editor)
5332        };
5333
5334        // Plain events only: run records (Program 2) stand for
5335        // `FormulaAdjusted` events, which have no forward effect here but
5336        // topology invalidation.
5337        let new_events = capture.events()[start_len..].to_vec();
5338        let new_runs = capture.lazy_len() > lazy_start;
5339        if new_events.iter().any(|event| {
5340            matches!(
5341                event,
5342                ChangeEvent::DefineName { .. }
5343                    | ChangeEvent::UpdateName { .. }
5344                    | ChangeEvent::DeleteName { .. }
5345            )
5346        }) {
5347            let all = capture.expanded_events_from(start_len, lazy_start);
5348            self.rollback_from_change_events(&all, invalidation_baseline)?;
5349            return Err(crate::engine::EditorError::TransactionUnsupported {
5350                reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
5351            });
5352        }
5353
5354        // Mirror value-impacting graph events to Arrow for forward edits.
5355        // This keeps Arrow overlays (delta + computed) consistent when edits clear/commit spills.
5356        self.clear_logged_cell_format_states(&new_events);
5357        for ev in &new_events {
5358            self.mirror_forward_change_to_arrow(ev);
5359        }
5360        for ev in &new_events {
5361            self.record_change_for_event(ev);
5362        }
5363
5364        // Atomic EngineAction calls publish one invalidation for their complete
5365        // journal at commit/rollback. Direct logged edits publish here.
5366        if self.action_depth == 0 {
5367            let mut impact =
5368                Self::classify_change_events(&new_events, LoggedEditDirection::Original);
5369            if new_runs {
5370                impact = impact.max(LoggedEditImpact::Topology);
5371            }
5372            self.apply_logged_edit_impact(impact, invalidation_baseline);
5373        }
5374
5375        Ok(ret)
5376    }
5377
5378    pub(crate) fn preflight_replay_admission(
5379        &mut self,
5380        events: &[ChangeEvent],
5381        forward: bool,
5382    ) -> Result<(), crate::engine::EditorError> {
5383        if !self.graph_admission_enabled() {
5384            return Ok(());
5385        }
5386        let mut vertex_delta = 0i128;
5387        let mut edge_delta = 0i128;
5388        let mut added_vertices = 0usize;
5389        let mut added_edges = 0usize;
5390        let mut formula_cells = BTreeSet::new();
5391        for event in events {
5392            match event {
5393                ChangeEvent::AddVertex {
5394                    formula,
5395                    coord,
5396                    sheet_id,
5397                    ..
5398                } => {
5399                    let delta = if forward { 1 } else { -1 };
5400                    vertex_delta += delta;
5401                    if forward {
5402                        added_vertices = added_vertices.saturating_add(1);
5403                        if formula.is_some() {
5404                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5405                        }
5406                    }
5407                }
5408                ChangeEvent::RemoveVertex {
5409                    old_formula,
5410                    coord,
5411                    sheet_id,
5412                    ..
5413                } => {
5414                    let delta = if forward { -1 } else { 1 };
5415                    vertex_delta += delta;
5416                    if !forward {
5417                        added_vertices = added_vertices.saturating_add(1);
5418                        if old_formula.is_some()
5419                            && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
5420                        {
5421                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5422                        }
5423                    }
5424                }
5425                ChangeEvent::EdgeAdded { .. } => {
5426                    let delta = if forward { 1 } else { -1 };
5427                    edge_delta += delta;
5428                    if forward {
5429                        added_edges = added_edges.saturating_add(1);
5430                    }
5431                }
5432                ChangeEvent::EdgeRemoved { .. } => {
5433                    let delta = if forward { -1 } else { 1 };
5434                    edge_delta += delta;
5435                    if !forward {
5436                        added_edges = added_edges.saturating_add(1);
5437                    }
5438                }
5439                ChangeEvent::SetFormula {
5440                    addr, old_formula, ..
5441                } if (forward || old_formula.is_some()) => {
5442                    formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
5443                }
5444                _ => {}
5445            }
5446        }
5447        let stats = self.graph.baseline_stats();
5448        let final_vertices = i128::try_from(stats.graph_vertex_count)
5449            .ok()
5450            .and_then(|count| count.checked_add(vertex_delta))
5451            .and_then(|count| usize::try_from(count).ok())
5452            .ok_or_else(|| {
5453                crate::engine::EditorError::Excel(
5454                    ExcelError::new(ExcelErrorKind::NImpl)
5455                        .with_message("replay vertex count overflow"),
5456                )
5457            })?;
5458        let final_edges = i128::try_from(stats.graph_edge_count)
5459            .ok()
5460            .and_then(|count| count.checked_add(edge_delta))
5461            .and_then(|count| usize::try_from(count).ok())
5462            .ok_or_else(|| {
5463                crate::engine::EditorError::Excel(
5464                    ExcelError::new(ExcelErrorKind::NImpl)
5465                        .with_message("replay edge count overflow"),
5466                )
5467            })?;
5468        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5469            final_vertices,
5470            final_edges,
5471            materialization_cells: formula_cells.len() as u64,
5472            added_vertices,
5473            added_edges,
5474        })
5475        .map_err(crate::engine::EditorError::Excel)
5476    }
5477
5478    /// Undo the last group still retained by the provided audit log.
5479    ///
5480    /// Disabled, zero-cap, and evicted history is unavailable on this index-based path. Use an
5481    /// explicit `ActionJournal` with `undo_action` when undo must be independent of audit retention.
5482    pub fn undo_logged(
5483        &mut self,
5484        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5485        log: &mut crate::engine::ChangeLog,
5486    ) -> Result<(), crate::engine::EditorError> {
5487        let pending_events = log
5488            .last_group_indices()
5489            .into_iter()
5490            .map(|index| log.events()[index].clone())
5491            .collect::<Vec<_>>();
5492        self.preflight_replay_admission(&pending_events, false)?;
5493        let invalidation_baseline = self.invalidation_baseline();
5494        // UndoEngine can fail after partially applying the batch, so publish
5495        // invalidation before replay rather than only on the success path.
5496        self.invalidate_for_change_events(
5497            &pending_events,
5498            LoggedEditDirection::InverseReplay,
5499            invalidation_baseline,
5500        );
5501        let batch = undo.undo(&mut self.graph, log)?;
5502        for item in batch.iter().rev() {
5503            self.apply_inverse_row_visibility_event(&item.event);
5504            self.apply_inverse_staged_formula_event(&item.event);
5505        }
5506        if !batch.is_empty() {
5507            let events = batch
5508                .iter()
5509                .map(|item| item.event.clone())
5510                .collect::<Vec<_>>();
5511            self.clear_logged_cell_format_states(&events);
5512        }
5513        self.mirror_undo_batch_to_arrow(&batch);
5514        if !batch.is_empty() {
5515            for item in &batch {
5516                self.record_change_for_event(&item.event);
5517            }
5518        }
5519        crate::engine::trace::fz_event!(
5520            tracing::Level::INFO,
5521            "history",
5522            "history.replay",
5523            op = "undo",
5524            events_replayed = batch.len(),
5525            ownership_resyncs = batch.len()
5526        );
5527        Ok(())
5528    }
5529
5530    pub fn redo_logged(
5531        &mut self,
5532        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5533        log: &mut crate::engine::ChangeLog,
5534    ) -> Result<(), crate::engine::EditorError> {
5535        let pending_events = undo.pending_redo_events();
5536        self.preflight_replay_admission(&pending_events, true)?;
5537        let invalidation_baseline = self.invalidation_baseline();
5538        self.invalidate_for_change_events(
5539            &pending_events,
5540            LoggedEditDirection::ForwardReplay,
5541            invalidation_baseline,
5542        );
5543        let batch = undo.redo(&mut self.graph, log)?;
5544        for item in &batch {
5545            self.apply_forward_row_visibility_event(&item.event);
5546            self.apply_forward_staged_formula_event(&item.event);
5547        }
5548        if !batch.is_empty() {
5549            let events = batch
5550                .iter()
5551                .map(|item| item.event.clone())
5552                .collect::<Vec<_>>();
5553            self.clear_logged_cell_format_states(&events);
5554        }
5555        self.mirror_redo_batch_to_arrow(&batch);
5556        if !batch.is_empty() {
5557            for item in &batch {
5558                self.record_change_for_event(&item.event);
5559            }
5560        }
5561        crate::engine::trace::fz_event!(
5562            tracing::Level::INFO,
5563            "history",
5564            "history.replay",
5565            op = "redo",
5566            events_replayed = batch.len(),
5567            ownership_resyncs = batch.len()
5568        );
5569        Ok(())
5570    }
5571
5572    /// Undo the last committed atomic action using the journal stack.
5573    ///
5574    /// This path does not require a `ChangeLog`.
5575    pub fn undo_action(
5576        &mut self,
5577        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5578    ) -> Result<(), crate::engine::EditorError> {
5579        let Some(journal) = undo.pop_undo_action() else {
5580            return Ok(());
5581        };
5582        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5583            undo.push_done_action(journal);
5584            return Err(error);
5585        }
5586        let invalidation_baseline = self.invalidation_baseline();
5587
5588        self.invalidate_for_action_journal(
5589            &journal,
5590            LoggedEditDirection::InverseReplay,
5591            invalidation_baseline,
5592        );
5593        journal.graph.undo(&mut self.graph)?;
5594        self.apply_inverse_row_visibility_events(&journal.graph.events);
5595        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
5596        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5597            for event in &journal.graph.events {
5598                self.record_change_for_event(event);
5599            }
5600        }
5601
5602        #[cfg(feature = "tracing")]
5603        let events_replayed = journal.graph.events.len();
5604        crate::engine::trace::fz_event!(
5605            tracing::Level::INFO,
5606            "history",
5607            "history.replay",
5608            op = "undo",
5609            events_replayed,
5610            ownership_resyncs = events_replayed
5611        );
5612        undo.push_redo_action(journal);
5613        Ok(())
5614    }
5615
5616    /// Redo the last undone atomic action using the journal stack.
5617    ///
5618    /// This path does not require a `ChangeLog`.
5619    pub fn redo_action(
5620        &mut self,
5621        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5622    ) -> Result<(), crate::engine::EditorError> {
5623        let Some(journal) = undo.pop_redo_action() else {
5624            return Ok(());
5625        };
5626        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5627            undo.push_redo_action(journal);
5628            return Err(error);
5629        }
5630        let invalidation_baseline = self.invalidation_baseline();
5631        self.invalidate_for_action_journal(
5632            &journal,
5633            LoggedEditDirection::ForwardReplay,
5634            invalidation_baseline,
5635        );
5636        journal.graph.redo(&mut self.graph)?;
5637        self.apply_forward_row_visibility_events(&journal.graph.events);
5638        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ false);
5639        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5640            for event in &journal.graph.events {
5641                self.record_change_for_event(event);
5642            }
5643        }
5644
5645        #[cfg(feature = "tracing")]
5646        let events_replayed = journal.graph.events.len();
5647        crate::engine::trace::fz_event!(
5648            tracing::Level::INFO,
5649            "history",
5650            "history.replay",
5651            op = "redo",
5652            events_replayed,
5653            ownership_resyncs = events_replayed
5654        );
5655        undo.push_done_action(journal);
5656        Ok(())
5657    }
5658
5659    fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5660        let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5661        // Coord stores 0-based indices.
5662        let row = addr.coord.row() + 1;
5663        let col = addr.coord.col() + 1;
5664        (sheet, row, col)
5665    }
5666
5667    fn mirror_undo_batch_to_arrow(
5668        &mut self,
5669        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5670    ) {
5671        // Undo applies inverses in reverse order.
5672        for item in batch.iter().rev() {
5673            self.mirror_inverse_change_to_arrow(&item.event);
5674        }
5675    }
5676
5677    fn mirror_redo_batch_to_arrow(
5678        &mut self,
5679        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5680    ) {
5681        // Redo applies events in forward order.
5682        for item in batch.iter() {
5683            self.mirror_forward_change_to_arrow(&item.event);
5684        }
5685    }
5686
5687    fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5688        use crate::engine::ChangeEvent;
5689        use formualizer_common::LiteralValue;
5690
5691        match ev {
5692            ChangeEvent::SetValue {
5693                addr,
5694                old_value,
5695                old_formula,
5696                ..
5697            } => {
5698                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5699                if old_formula.is_some() {
5700                    self.clear_delta_overlay_cell(&sheet, row, col);
5701                } else {
5702                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5703                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5704                }
5705            }
5706            ChangeEvent::SetFormula {
5707                addr,
5708                old_value,
5709                old_formula,
5710                ..
5711            } => {
5712                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5713                if old_formula.is_some() {
5714                    self.clear_delta_overlay_cell(&sheet, row, col);
5715                } else {
5716                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5717                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5718                }
5719            }
5720            ChangeEvent::SpillCommitted { old, new, .. } => {
5721                // Inverse: restore `old` (or clear if none).
5722                self.mirror_spill_snapshot(new, /*clear_only=*/ true);
5723                if let Some(snap) = old {
5724                    self.mirror_spill_snapshot(snap, /*clear_only=*/ false);
5725                }
5726            }
5727            ChangeEvent::SpillCleared { old, .. } => {
5728                // Inverse: restore prior spill.
5729                self.mirror_spill_snapshot(old, /*clear_only=*/ false);
5730            }
5731            ChangeEvent::SetRowVisibility { .. } => {
5732                // Engine-side metadata only; no Arrow overlay effect.
5733            }
5734            _ => {}
5735        }
5736    }
5737
5738    fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5739        use crate::engine::ChangeEvent;
5740
5741        match ev {
5742            ChangeEvent::SetValue { addr, new, .. } => {
5743                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5744                self.mirror_value_to_overlay(&sheet, row, col, new);
5745            }
5746            ChangeEvent::SetFormula { addr, .. } => {
5747                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5748                self.clear_delta_overlay_cell(&sheet, row, col);
5749                // Keep any computed overlay for this cell as-is; it will be recomputed on demand.
5750            }
5751            ChangeEvent::SpillCommitted { old, new, .. } => {
5752                if let Some(snap) = old {
5753                    self.mirror_spill_snapshot(snap, /*clear_only=*/ true);
5754                }
5755                self.mirror_spill_snapshot(new, /*clear_only=*/ false);
5756            }
5757            ChangeEvent::SpillCleared { old, .. } => {
5758                self.mirror_spill_snapshot(old, /*clear_only=*/ true);
5759            }
5760            ChangeEvent::SetRowVisibility { .. } => {
5761                // Engine-side metadata only; no Arrow overlay effect.
5762            }
5763            _ => {
5764                // Other graph structural operations do not have direct value effects in Arrow.
5765            }
5766        }
5767    }
5768
5769    fn mirror_spill_snapshot(
5770        &mut self,
5771        snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5772        clear_only: bool,
5773    ) {
5774        use formualizer_common::LiteralValue;
5775
5776        let mut i = 0usize;
5777        for row in &snap.values {
5778            for v in row {
5779                if let Some(cell) = snap.target_cells.get(i) {
5780                    let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5781                    let out = if clear_only {
5782                        LiteralValue::Empty
5783                    } else {
5784                        v.clone()
5785                    };
5786                    self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5787                }
5788                i += 1;
5789            }
5790        }
5791        // If target_cells is longer than values (should not happen), clear remaining cells.
5792        if clear_only {
5793            for cell in snap.target_cells.iter().skip(i) {
5794                let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5795                self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5796            }
5797        }
5798    }
5799
5800    pub fn set_default_sheet_by_name(&mut self, name: &str) {
5801        self.graph.set_default_sheet_by_name(name);
5802    }
5803
5804    pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5805        self.graph.set_default_sheet_by_id(id);
5806    }
5807
5808    pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5809        self.graph.set_sheet_index_mode(mode);
5810    }
5811
5812    #[cfg(feature = "test-support")]
5813    #[doc(hidden)]
5814    pub fn mark_all_formulas_dirty_for_test(&mut self) {
5815        self.mark_all_formula_vertices_dirty();
5816    }
5817
5818    #[cfg(feature = "test-support")]
5819    #[doc(hidden)]
5820    pub fn take_criteria_mask_work_for_test() -> (usize, usize) {
5821        criteria_mask_test_hooks::take_mask_work()
5822    }
5823
5824    #[cfg(feature = "test-support")]
5825    #[doc(hidden)]
5826    pub fn lookup_index_cache_report_for_test(&self) -> LookupIndexCacheReport {
5827        self.lookup_index_cache.report()
5828    }
5829
5830    #[cfg(any(test, feature = "benchmark_internal"))]
5831    #[doc(hidden)]
5832    pub fn reset_recalc_reuse_probe(&mut self) {
5833        *self.recalc_reuse_probe.get_mut().unwrap() = RecalcReuseProbe::default();
5834    }
5835
5836    #[cfg(any(test, feature = "benchmark_internal"))]
5837    #[doc(hidden)]
5838    pub fn recalc_reuse_probe(&self) -> RecalcReuseProbe {
5839        let mut probe = self.recalc_reuse_probe.lock().unwrap().clone();
5840        if let Some(cached) = self.cached_static_schedule.as_ref() {
5841            let entry_bytes = |e: &CachedScheduleEntry| {
5842                std::mem::size_of::<CachedScheduleEntry>()
5843                    + e.candidate_vertices.heap_bytes()
5844                    + std::mem::size_of::<crate::engine::Schedule>()
5845                    + 2 * std::mem::size_of::<usize>()
5846                    + schedule_probe_retained_bytes(&e.schedule)
5847            };
5848            probe.schedule_retained_bytes = entry_bytes(cached)
5849                + self.recent_schedules.iter().map(entry_bytes).sum::<usize>()
5850                + self.base_schedule.as_ref().map_or(0, entry_bytes)
5851                + self.recent_schedules.capacity() * std::mem::size_of::<CachedScheduleEntry>();
5852        }
5853        probe
5854    }
5855
5856    #[cfg(test)]
5857    pub(crate) fn cached_static_schedule_for_test(&self) -> Option<Arc<crate::engine::Schedule>> {
5858        self.cached_static_schedule
5859            .as_ref()
5860            .map(|cached| Arc::clone(&cached.schedule))
5861    }
5862
5863    fn clear_cached_static_schedule(&mut self) {
5864        self.cached_static_schedule = None;
5865        self.recent_schedules.clear();
5866        self.base_schedule = None;
5867    }
5868
5869    /// Keep a replaced schedule among the recent ones when it is still
5870    /// current and small; drop stale ones and the oldest beyond the bounds.
5871    fn retain_recent_schedule(&mut self, entry: CachedScheduleEntry) {
5872        const RECENT_SCHEDULES: usize = 8;
5873        const RECENT_SCHEDULE_VERTICES: usize = 65_536;
5874        let revision = self.schedule_cache_authority_revision();
5875        let epoch = self.topology_epoch;
5876        self.recent_schedules
5877            .retain(|e| e.topology_epoch == epoch && e.authority_revision == revision);
5878        let current =
5879            |e: &CachedScheduleEntry| e.topology_epoch == epoch && e.authority_revision == revision;
5880        if self.base_schedule.as_ref().is_some_and(|b| !current(b)) {
5881            self.base_schedule = None;
5882        }
5883        // The largest current schedule becomes the base; a replaced base
5884        // may still join the recent ones.
5885        let mut entry = entry;
5886        if current(&entry)
5887            && entry.candidate_vertices.len() > BASE_SCHEDULE_MIN_VERTICES
5888            && self
5889                .base_schedule
5890                .as_ref()
5891                .is_none_or(|b| entry.candidate_vertices.len() > b.candidate_vertices.len())
5892        {
5893            match self.base_schedule.replace(entry) {
5894                Some(previous) => entry = previous,
5895                None => return,
5896            }
5897        }
5898        if entry.topology_epoch != epoch
5899            || entry.authority_revision != revision
5900            || entry.candidate_vertices.len() > RECENT_SCHEDULE_VERTICES / 4
5901        {
5902            return;
5903        }
5904        self.recent_schedules.push(entry);
5905        let mut total: usize = self
5906            .recent_schedules
5907            .iter()
5908            .map(|e| e.candidate_vertices.len())
5909            .sum();
5910        while self.recent_schedules.len() > RECENT_SCHEDULES || total > RECENT_SCHEDULE_VERTICES {
5911            let oldest = self.recent_schedules.remove(0);
5912            total -= oldest.candidate_vertices.len();
5913        }
5914    }
5915
5916    fn invalidation_baseline(&self) -> InvalidationBaseline {
5917        InvalidationBaseline {
5918            snapshot_id: self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed),
5919            topology_epoch: self.topology_epoch,
5920        }
5921    }
5922
5923    fn classify_change_events(
5924        events: &[crate::engine::ChangeEvent],
5925        direction: LoggedEditDirection,
5926    ) -> LoggedEditImpact {
5927        use crate::engine::ChangeEvent;
5928
5929        events
5930            .iter()
5931            .map(|event| match event {
5932                ChangeEvent::CompoundStart { .. } | ChangeEvent::CompoundEnd { .. } => {
5933                    LoggedEditImpact::NoOp
5934                }
5935                ChangeEvent::SetRowVisibility { .. }
5936                | ChangeEvent::SetValue {
5937                    old_formula: None,
5938                    old_value: Some(_),
5939                    ..
5940                } => LoggedEditImpact::DataOnly,
5941                // Original canonical writes can lack an Arrow old value while
5942                // updating an existing placeholder. Inverse replay actually
5943                // removes that vertex; a later redo recreates it.
5944                ChangeEvent::SetValue {
5945                    old_formula: None,
5946                    old_value: None,
5947                    ..
5948                } if direction == LoggedEditDirection::Original => LoggedEditImpact::DataOnly,
5949                ChangeEvent::SetValue { .. }
5950                | ChangeEvent::SetFormula { .. }
5951                | ChangeEvent::AddVertex { .. }
5952                | ChangeEvent::RemoveVertex { .. }
5953                | ChangeEvent::VertexMoved { .. }
5954                | ChangeEvent::FormulaAdjusted { .. }
5955                | ChangeEvent::NamedRangeAdjusted { .. }
5956                | ChangeEvent::EdgeAdded { .. }
5957                | ChangeEvent::EdgeRemoved { .. }
5958                | ChangeEvent::DefineName { .. }
5959                | ChangeEvent::UpdateName { .. }
5960                | ChangeEvent::DeleteName { .. }
5961                | ChangeEvent::SpillCommitted { .. }
5962                | ChangeEvent::SpillCleared { .. }
5963                | ChangeEvent::StagedFormulaCellChanged { .. } => LoggedEditImpact::Topology,
5964            })
5965            .max()
5966            .unwrap_or(LoggedEditImpact::NoOp)
5967    }
5968
5969    fn classify_arrow_undo(arrow: &crate::engine::ArrowUndoBatch) -> LoggedEditImpact {
5970        use crate::engine::ArrowOp;
5971
5972        arrow
5973            .ops
5974            .iter()
5975            .map(|op| match op {
5976                ArrowOp::SetDeltaCell { .. } | ArrowOp::SetComputedCell { .. } => {
5977                    LoggedEditImpact::DataOnly
5978                }
5979                ArrowOp::RestoreComputedRect { .. }
5980                | ArrowOp::InsertRows { .. }
5981                | ArrowOp::InsertCols { .. } => LoggedEditImpact::Topology,
5982            })
5983            .max()
5984            .unwrap_or(LoggedEditImpact::NoOp)
5985    }
5986
5987    fn apply_logged_edit_impact(
5988        &mut self,
5989        impact: LoggedEditImpact,
5990        baseline: InvalidationBaseline,
5991    ) {
5992        match impact {
5993            LoggedEditImpact::NoOp => {}
5994            LoggedEditImpact::DataOnly => {
5995                if self.topology_epoch == baseline.topology_epoch
5996                    && self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
5997                        == baseline.snapshot_id
5998                {
5999                    self.mark_data_edited();
6000                }
6001            }
6002            LoggedEditImpact::Topology => {
6003                // Some structural entry points already publish topology
6004                // invalidation. Do not bump the same batch twice.
6005                if self.topology_epoch == baseline.topology_epoch {
6006                    self.mark_topology_edited();
6007                }
6008            }
6009        }
6010    }
6011
6012    fn invalidate_for_change_events(
6013        &mut self,
6014        events: &[crate::engine::ChangeEvent],
6015        direction: LoggedEditDirection,
6016        baseline: InvalidationBaseline,
6017    ) {
6018        self.apply_logged_edit_impact(Self::classify_change_events(events, direction), baseline);
6019    }
6020
6021    fn invalidate_for_action_journal(
6022        &mut self,
6023        journal: &crate::engine::ActionJournal,
6024        direction: LoggedEditDirection,
6025        baseline: InvalidationBaseline,
6026    ) {
6027        let impact = Self::classify_change_events(&journal.graph.events, direction)
6028            .max(Self::classify_arrow_undo(&journal.arrow));
6029        self.apply_logged_edit_impact(impact, baseline);
6030    }
6031
6032    /// Mark data edited: bump snapshot and set edited flag.
6033    /// Value-only edits keep the stable-topology schedule cache alive.
6034    pub fn mark_data_edited(&mut self) {
6035        self.lookup_index_cache.clear();
6036        self.snapshot_id
6037            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6038        self.has_edited = true;
6039    }
6040
6041    /// Mark a topology-changing edit: bump snapshot + topology epoch and invalidate cached schedules.
6042    pub fn mark_topology_edited(&mut self) {
6043        self.arrow_sheet_index.take();
6044        self.lookup_index_cache.clear();
6045        self.snapshot_id
6046            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6047        self.topology_epoch = self.topology_epoch.wrapping_add(1);
6048        self.graph.bump_topology_revision();
6049        self.clear_cached_static_schedule();
6050        if let Some(ledger) = self.active_resource_ledger.as_mut() {
6051            let released = ledger.account_mixed_cache(0);
6052            debug_assert!(released.is_ok());
6053        }
6054        self.has_edited = true;
6055        // Eager sync at a topology edit, so read-only (`&self`) plans and
6056        // inspection see a current authority (not during a load or an open
6057        // structural capture).
6058        self.graph.authority_sync_eager();
6059    }
6060
6061    fn mark_all_formula_vertices_dirty(&mut self) {
6062        let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
6063        for vertex in vertices {
6064            self.graph.mark_vertex_dirty(vertex);
6065        }
6066    }
6067
6068    fn mark_moved_formula_vertices_dirty(
6069        &mut self,
6070        summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
6071    ) {
6072        for vertex in &summary.vertices_moved {
6073            if self.graph.has_formula(*vertex) {
6074                self.graph.mark_vertex_dirty(*vertex);
6075            }
6076        }
6077    }
6078
6079    /// Access Arrow sheet store (read-only)
6080    pub fn sheet_store(&self) -> &SheetStore {
6081        &self.arrow_sheets
6082    }
6083
6084    /// True when any sheet carries manual/filter row-visibility state.
6085    /// Used by load-time freshness checks (see `Engine::adopt_file_sheets`).
6086    pub(crate) fn has_row_visibility_state(&self) -> bool {
6087        !self.row_visibility.is_empty()
6088    }
6089
6090    /// Access Arrow sheet store (mutable)
6091    pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
6092        self.arrow_sheet_index.take();
6093        &mut self.arrow_sheets
6094    }
6095
6096    pub fn has_staged_formulas(&self) -> bool {
6097        !self.staged_formulas.is_empty()
6098    }
6099
6100    pub fn staged_formula_count(&self) -> usize {
6101        self.staged_formulas.values().map(StagedSheet::len).sum()
6102    }
6103
6104    /// Stage a formula text instead of inserting into the graph (used when deferring is enabled).
6105    pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
6106        self.staged_formulas
6107            .entry(sheet.to_string())
6108            .or_default()
6109            .stage(row, col, text);
6110        self.staged_formula_index.stage(sheet, row, col);
6111        if let Some(sheet) = self.graph.sheet_id(sheet) {
6112            self.invalidate_pending_spills(StructuralScope::Cell {
6113                sheet,
6114                row: row.saturating_sub(1),
6115                col: col.saturating_sub(1),
6116            });
6117        }
6118    }
6119
6120    fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
6121        for (row, col, _) in &staged.entries {
6122            self.staged_formula_index.remove(sheet, *row, *col);
6123        }
6124        if staged.deferred_package.is_some() {
6125            self.staged_formula_index.set_package(sheet, None);
6126        }
6127    }
6128
6129    fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
6130        self.staged_formulas.insert(sheet, staged);
6131    }
6132
6133    #[doc(hidden)]
6134    pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
6135        SourceFormulaIngress { engine: self }
6136    }
6137
6138    #[doc(hidden)]
6139    /// Test-only fault-injection seam. Not part of the supported API; it exists so
6140    /// integration tests in sibling crates can fail a commit at an exact point.
6141    #[doc(hidden)]
6142    #[cfg(any(test, feature = "test-support"))]
6143    pub fn set_before_prepared_span_commit_hook(
6144        &mut self,
6145        hook: impl FnOnce() + Send + Sync + 'static,
6146    ) {
6147        self.before_prepared_span_commit_hook = Some(Box::new(hook));
6148    }
6149
6150    #[doc(hidden)]
6151    /// Test-only fault-injection seam, matching `set_after_eager_proposal_commit_hook`.
6152    #[cfg(test)]
6153    pub(crate) fn set_before_target_preparation_commit_hook(
6154        &mut self,
6155        hook: impl FnOnce() + Send + Sync + 'static,
6156    ) {
6157        self.before_target_preparation_commit_hook = Some(Box::new(hook));
6158    }
6159
6160    fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
6161        if let Ok(replay) = package.replay.lock()
6162            && let Some(footprint) = replay.selection_cache_footprint()
6163            && !self
6164                .source_cache_footprints
6165                .iter()
6166                .any(|known| known.ptr_eq(&footprint))
6167        {
6168            self.source_cache_footprints.push(footprint);
6169        }
6170        let sheet = package.sheet_name.clone();
6171        let staged = self.staged_formulas.entry(sheet.clone()).or_default();
6172        debug_assert!(staged.deferred_package.is_none());
6173        staged.deferred_package = Some(package);
6174        staged.reconcile_attached_deferred_package();
6175        self.staged_formula_index
6176            .set_package(&sheet, staged.deferred_package.as_ref());
6177        if let Some(sheet_id) = self.graph.sheet_id(&sheet) {
6178            let package = self
6179                .staged_formulas
6180                .get(&sheet)
6181                .and_then(|staged| staged.deferred_package.as_ref());
6182            for &(vertex, anchor, region) in &self.blocked_pending_spills {
6183                if anchor.sheet_id == sheet_id
6184                    && self.graph.vertex_exists(vertex)
6185                    && self.graph.get_cell_ref(vertex) == Some(anchor)
6186                    && self.staged_formula_index.package_occupies_spill(
6187                        &sheet,
6188                        (anchor.coord.row() + 1, anchor.coord.col() + 1),
6189                        (
6190                            region.rows.query_bounds().1 + 1,
6191                            region.cols.query_bounds().1 + 1,
6192                        ),
6193                        |point| package.is_some_and(|package| package.suppressed.contains(&point)),
6194                    )
6195                {
6196                    self.graph.mark_vertex_dirty(vertex);
6197                }
6198            }
6199        }
6200    }
6201
6202    pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
6203        let mut removed = None;
6204        let mut remove_sheet = false;
6205        let mut had_package = false;
6206        if let Some(entries) = self.staged_formulas.get_mut(sheet) {
6207            had_package = entries.deferred_package.is_some();
6208            removed = entries.remove(row, col);
6209            remove_sheet = entries.is_empty();
6210        }
6211        let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
6212        if !ordinary_removed && had_package {
6213            self.staged_formula_index.touch_package(sheet);
6214        }
6215        if remove_sheet {
6216            self.staged_formulas.remove(sheet);
6217            self.staged_formula_index.set_package(sheet, None);
6218        }
6219        if (ordinary_removed || had_package)
6220            && let Some(sheet) = self.graph.sheet_id(sheet)
6221        {
6222            self.invalidate_pending_spills(StructuralScope::Cell {
6223                sheet,
6224                row: row.saturating_sub(1),
6225                col: col.saturating_sub(1),
6226            });
6227        }
6228        removed
6229    }
6230
6231    pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
6232        if self.staged_formulas.remove(sheet).is_some() {
6233            self.staged_formula_index.clear_sheet(sheet);
6234            if let Some(sheet_id) = self.graph.sheet_id(sheet) {
6235                self.invalidate_pending_spills(StructuralScope::Sheet(sheet_id));
6236            }
6237        }
6238    }
6239
6240    pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
6241        let Some(entries) = self.staged_formulas.remove(old) else {
6242            return;
6243        };
6244        self.staged_formula_index.clear_sheet(old);
6245        let (formulas, mut package) = entries.into_parts();
6246        for (row, col, text) in formulas {
6247            self.stage_formula_text(new, row, col, text);
6248        }
6249        if let Some(package) = package.as_mut() {
6250            package.sheet_name = new.to_string();
6251        }
6252        if let Some(package) = package {
6253            self.stage_deferred_formula_package(package);
6254        }
6255    }
6256
6257    /// Get a staged formula text for a given cell if present (cloned).
6258    pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
6259        self.staged_formulas
6260            .get(sheet)
6261            .and_then(|v| v.get(row, col))
6262    }
6263
6264    pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
6265        &self.formula_parse_diagnostics
6266    }
6267
6268    pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
6269        std::mem::take(&mut self.formula_parse_diagnostics)
6270    }
6271
6272    pub fn clear_formula_parse_diagnostics(&mut self) {
6273        self.formula_parse_diagnostics.clear();
6274    }
6275
6276    pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
6277        self.last_formula_ingest_report.as_ref()
6278    }
6279
6280    pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
6281        &self.formula_ingest_report_total
6282    }
6283
6284    #[cfg(test)]
6285    pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
6286        &mut self,
6287        hook: impl FnOnce() + Send + Sync + 'static,
6288    ) {
6289        self.before_target_planning_snapshot_hook = Some(Box::new(hook));
6290    }
6291
6292    #[cfg(test)]
6293    pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
6294        self.inject_target_semantic_stale_once_for_test = true;
6295    }
6296
6297    #[cfg(test)]
6298    pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
6299        self.force_virtual_dep_changes_remaining_for_test = rounds;
6300    }
6301
6302    #[cfg(test)]
6303    pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
6304        self.fail_evaluation_commit_preflight_once_for_test = true;
6305    }
6306
6307    #[cfg(test)]
6308    pub(crate) fn set_target_preparation_fault_for_test(
6309        &mut self,
6310        fault: crate::engine::target_preparation::TargetPreparationFault,
6311    ) {
6312        self.target_preparation_fault_for_test = Some(fault);
6313    }
6314
6315    #[cfg(test)]
6316    pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
6317        self.staged_formula_index.revision()
6318    }
6319
6320    #[cfg(test)]
6321    pub(crate) fn deferred_package_for_test(
6322        &self,
6323        sheet: &str,
6324    ) -> &crate::engine::DeferredFormulaPackage {
6325        self.staged_formulas
6326            .get(sheet)
6327            .unwrap()
6328            .deferred_package
6329            .as_ref()
6330            .unwrap()
6331    }
6332
6333    #[cfg(test)]
6334    pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
6335        let ordinary_storage = self
6336            .staged_formulas
6337            .values()
6338            .map(|sheet| sheet.entries.len())
6339            .sum::<usize>();
6340        let package_storage = self
6341            .staged_formulas
6342            .values()
6343            .filter(|sheet| sheet.deferred_package.is_some())
6344            .count();
6345        ordinary_storage == self.staged_formula_index.ordinary_count()
6346            && package_storage == self.staged_formula_index.package_count()
6347            && self.staged_formulas.iter().all(|(name, sheet)| {
6348                sheet.entries.iter().all(|(row, col, _)| {
6349                    let leases = self
6350                        .staged_formula_index
6351                        .leases_in_region(name, *row, *col, *row, *col);
6352                    leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
6353                })
6354            })
6355    }
6356
6357    #[cfg(test)]
6358    pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
6359        self.evaluation_request_begin_count_for_test
6360    }
6361
6362    #[cfg(test)]
6363    pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
6364        &mut self,
6365        hook: impl FnOnce() + Send + Sync + 'static,
6366    ) {
6367        self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
6368    }
6369
6370    #[cfg(test)]
6371    pub(crate) fn set_after_eager_proposal_commit_hook(
6372        &mut self,
6373        hook: impl FnOnce() + Send + Sync + 'static,
6374    ) {
6375        self.after_eager_proposal_commit_hook = Some(Box::new(hook));
6376    }
6377
6378    #[cfg(test)]
6379    pub(crate) fn topology_epoch_for_test(&self) -> u64 {
6380        self.topology_epoch
6381    }
6382
6383    #[cfg(test)]
6384    pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
6385        self.graph.topology_revision()
6386    }
6387
6388    fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
6389        self.formula_ingest_report_total.mode = report.mode;
6390        self.formula_ingest_report_total.accumulate(&report);
6391        self.last_formula_ingest_report = Some(report);
6392    }
6393
6394    fn collect_planning_function_requests(
6395        ast: &ASTNode,
6396        requests: &mut Vec<(String, String, usize)>,
6397    ) {
6398        match &ast.node_type {
6399            ASTNodeType::Function { name, args } => {
6400                requests.push((String::new(), name.clone(), args.len()));
6401                for arg in args {
6402                    Self::collect_planning_function_requests(arg, requests);
6403                }
6404            }
6405            ASTNodeType::BinaryOp { left, right, .. } => {
6406                Self::collect_planning_function_requests(left, requests);
6407                Self::collect_planning_function_requests(right, requests);
6408            }
6409            ASTNodeType::UnaryOp { expr, .. } => {
6410                Self::collect_planning_function_requests(expr, requests);
6411            }
6412            ASTNodeType::Call { callee, args } => {
6413                Self::collect_planning_function_requests(callee, requests);
6414                for arg in args {
6415                    Self::collect_planning_function_requests(arg, requests);
6416                }
6417            }
6418            ASTNodeType::Array(rows) => {
6419                for cell in rows.iter().flatten() {
6420                    Self::collect_planning_function_requests(cell, requests);
6421                }
6422            }
6423            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6424        }
6425    }
6426
6427    fn prepared_function_semantics_changed(
6428        &self,
6429        preparation: &crate::engine::FormulaCompressedPreparation,
6430        guard: &crate::function_registry::SemanticEpochReadGuard,
6431    ) -> bool {
6432        if preparation.function_semantic_epoch == guard.epoch() {
6433            return false;
6434        }
6435
6436        guard
6437            .semantic_changes_affect_requests_since(preparation.function_semantic_epoch, Vec::new())
6438    }
6439
6440    pub(crate) fn prepare_source_formula_families(
6441        &mut self,
6442        sheet_name: &str,
6443        families: &[crate::engine::SourceFormulaFamily],
6444    ) -> crate::engine::FormulaCompressedPreparation {
6445        crate::engine::FormulaCompressedPreparation {
6446            engine_token: Arc::clone(&self.source_formula_token),
6447            function_semantic_epoch: crate::function_registry::semantic_epoch(),
6448            function_provider_revision: None,
6449            function_semantics_used: false,
6450            sheet_name: Arc::from(sheet_name),
6451            rejected: BTreeMap::new(),
6452            eager_replay: Vec::new(),
6453            preparation_spool_replays: 0,
6454            clean_rejected_anchor_counts: [0; 3],
6455            fragmented_rejected_anchor_counts: [0; 3],
6456            exact_replay: None,
6457            replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6458        }
6459    }
6460
6461    fn prepare_source_formula_proposals(
6462        &mut self,
6463        sheet_name: &str,
6464        families: &[crate::engine::SourceFormulaFamily],
6465        authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6466        replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6467        formula_record_count: u64,
6468        replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6469        suppressed: &BTreeSet<(u32, u32)>,
6470        consumed: &FxHashSet<(crate::engine::SourceFamilyId, crate::engine::SourceCoord)>,
6471        consumed_engine: Option<&Arc<()>>,
6472    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
6473        let mut preparation = self.prepare_source_formula_families(sheet_name, families);
6474        preparation.exact_replay = Some(Arc::clone(&replay));
6475        preparation
6476            .replay_disposition
6477            .register_consumed_members(consumed_engine, consumed.iter().copied());
6478        if !preparation
6479            .replay_disposition
6480            .consumed_engine_matches(&self.source_formula_token)
6481        {
6482            return Err(ExcelError::new(ExcelErrorKind::Value)
6483                .with_message("ResidualConsumedEngineMismatch"));
6484        }
6485        preparation
6486            .replay_disposition
6487            .extend_suppressed_excel_coords(suppressed.iter().copied());
6488        Ok(preparation)
6489    }
6490
6491    fn formula_batch_from_exact_replay(
6492        &mut self,
6493        sheet_name: &str,
6494        replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
6495    ) -> Result<FormulaIngestBatch, ExcelError> {
6496        let mut cache = rustc_hash::FxHashMap::default();
6497        let mut formulas = Vec::new();
6498        for record in replayed {
6499            let key = if record.text.starts_with('=') {
6500                record.text
6501            } else {
6502                format!("={}", record.text)
6503            };
6504            let ast_id = if let Some(cached) = cache.get(&key) {
6505                *cached
6506            } else {
6507                let parsed = match formualizer_parse::parser::parse(&key) {
6508                    Ok(parsed) => parsed,
6509                    Err(error) => {
6510                        let Some(parsed) = self.handle_formula_parse_error(
6511                            sheet_name,
6512                            record.row,
6513                            record.col,
6514                            &key,
6515                            error.to_string(),
6516                        )?
6517                        else {
6518                            continue;
6519                        };
6520                        parsed
6521                    }
6522                };
6523                let ast_id = self.intern_formula_ast(&parsed);
6524                cache.insert(key.clone(), ast_id);
6525                ast_id
6526            };
6527            formulas.push(
6528                FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
6529                    .with_source_proof(record.source_order, record.family, record.partition_owner),
6530            );
6531        }
6532        Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
6533    }
6534
6535    fn prepare_target_combined_legacy_graph(
6536        &self,
6537        packages: &[PreparedTargetSourcePackage],
6538        ordinary: &[PreparedOrdinaryStagedFormula],
6539    ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
6540        let mut planned_by_coord = BTreeMap::new();
6541        for package in packages {
6542            for (row, col, ast_id, plan) in &package.legacy {
6543                planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
6544            }
6545        }
6546        for formula in ordinary {
6547            if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
6548                planned_by_coord.insert(
6549                    (formula.sheet_id, formula.lease.row, formula.lease.col),
6550                    (ast_id, plan),
6551                );
6552            }
6553        }
6554        let planned = planned_by_coord
6555            .into_iter()
6556            .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
6557            .collect::<Vec<_>>();
6558        let formula_count = planned.len();
6559        let graph = self
6560            .graph
6561            .prepare_legacy_graph_plan_multi_sheet(planned)
6562            .map_err(|error| {
6563                ExcelError::new(ExcelErrorKind::Value)
6564                    .with_message(format!("target graph preparation failed: {error}"))
6565            })?;
6566        Ok((graph, formula_count))
6567    }
6568
6569    fn replay_target_coordinates(
6570        &mut self,
6571        replay: &Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6572        coordinates: &[(u32, u32)],
6573        deadline: Option<std::time::Instant>,
6574        scratch: &mut u64,
6575    ) -> Result<Option<Vec<crate::engine::DeferredReplayFormula>>, ExcelError> {
6576        if coordinates.is_empty() {
6577            return Ok(Some(Vec::new()));
6578        }
6579        let mut guard = replay.lock().map_err(|_| {
6580            ExcelError::new(ExcelErrorKind::Value)
6581                .with_message("deferred formula spool lock poisoned")
6582        })?;
6583        let retained = guard.selection_cache_footprint().is_some();
6584        let records = guard.replay_selected_exact(coordinates, &mut |work, bytes| {
6585            self.target_preparation_checkpoint(deadline, work)?;
6586            if retained && let Some(ledger) = self.active_resource_ledger.as_mut() {
6587                ledger
6588                    .reserve_retained(bytes)
6589                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
6590                self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
6591            }
6592            self.reserve_graph_source_scratch(bytes)?;
6593            *scratch = scratch.saturating_add(bytes);
6594            Ok(())
6595        });
6596        drop(guard);
6597        let reconciled = self.reconcile_source_cache_footprints();
6598        let records = records?;
6599        reconciled?;
6600        Ok(records)
6601    }
6602
6603    fn prepare_target_exact_source_selection(
6604        &mut self,
6605        sheet: &str,
6606        lease: StagedPackageLease,
6607        coordinates: Vec<(u32, u32)>,
6608        previous: &BTreeSet<(u32, u32)>,
6609        allow_partial_shared: bool,
6610        deadline: Option<std::time::Instant>,
6611        scratch: &mut u64,
6612    ) -> Result<Option<PreparedTargetSourcePackage>, ExcelError> {
6613        let package = self
6614            .staged_formulas
6615            .get(sheet)
6616            .and_then(|s| s.deferred_package.as_ref())
6617            .unwrap();
6618        if !package.source_geometry_complete
6619            || ((!package.families.is_empty() || !package.partitioned_families.is_empty())
6620                && !package.coordinates_cover_families)
6621            || package.reconciliation_replay.is_some()
6622        {
6623            return Ok(None);
6624        }
6625        let selected_points: BTreeSet<_> = coordinates
6626            .into_iter()
6627            .filter(|point| !package.suppressed.contains(point))
6628            .collect();
6629        let mut prepared = PreparedTargetSourcePackage::empty_selection(
6630            sheet,
6631            self.graph.sheet_id(sheet).unwrap(),
6632            lease,
6633        );
6634        if selected_points.is_empty() {
6635            return Ok(Some(prepared));
6636        }
6637        let mut routing = crate::engine::FormulaReplayDisposition::default();
6638        for partition in &package.partitioned_families {
6639            routing
6640                .register_partition(partition, false)
6641                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
6642        }
6643        routing.extend_suppressed_excel_coords(package.suppressed.iter().copied());
6644        routing.register_consumed_members(
6645            package.consumed_engine.as_ref(),
6646            package.consumed_members.iter().copied(),
6647        );
6648        let replay = Arc::clone(&package.replay);
6649        let source_report = package.accounting_report();
6650        prepared.source_report = source_report;
6651        prepared.disposition = routing.clone();
6652
6653        let replay_points: BTreeSet<_> = selected_points
6654            .iter()
6655            .copied()
6656            .filter(|&(row, col)| !prepared.direct_contains(row, col))
6657            .collect();
6658        let coordinates: Vec<_> = replay_points.iter().copied().collect();
6659        let records = self.replay_target_coordinates(&replay, &coordinates, deadline, scratch)?;
6660        prepared.spool_replays = u64::from(!coordinates.is_empty());
6661        let Some(mut replay_records) = records else {
6662            return Ok(None);
6663        };
6664        for record in &mut replay_records {
6665            if record.family.is_none() {
6666                record.partition_owner = routing
6667                    .ordinary_disposition(crate::engine::SourceCoord {
6668                        row: record.row.saturating_sub(1),
6669                        col: record.col.saturating_sub(1),
6670                    })
6671                    .1;
6672            }
6673        }
6674        replay_records.sort_by_key(|record| record.source_order);
6675        if replay_records
6676            .iter()
6677            .any(|record| !selected_points.contains(&(record.row, record.col)))
6678            || replay_records
6679                .windows(2)
6680                .any(|records| records[0].source_order == records[1].source_order)
6681        {
6682            return Err(ExcelError::new(ExcelErrorKind::Value)
6683                .with_message("invalid indexed exact source selection"));
6684        }
6685        let represented: BTreeSet<_> = replay_records.iter().map(|r| (r.row, r.col)).collect();
6686        if represented != replay_points {
6687            return Err(ExcelError::new(ExcelErrorKind::Value)
6688                .with_message("incomplete indexed exact source selection"));
6689        }
6690        // The last source record must agree with compressed ownership evidence.
6691        // Earlier overridden records retain ordering but demand no dependencies.
6692        let source = self
6693            .staged_formulas
6694            .get(sheet)
6695            .unwrap()
6696            .deferred_package
6697            .as_ref()
6698            .unwrap();
6699        let contains = |rect: crate::engine::SourceRect, coord: crate::engine::SourceCoord| {
6700            coord.row >= rect.start.row
6701                && coord.row <= rect.end.row
6702                && coord.col >= rect.start.col
6703                && coord.col <= rect.end.col
6704        };
6705        let mut checked = BTreeSet::new();
6706        for record in replay_records.iter().rev() {
6707            if !checked.insert((record.row, record.col)) {
6708                continue;
6709            }
6710            let coord = crate::engine::SourceCoord {
6711                row: record.row - 1,
6712                col: record.col - 1,
6713            };
6714            let agrees = |owner, shared: bool| {
6715                record.family == shared.then_some(owner)
6716                    && record.partition_owner.or(record.family) == Some(owner)
6717            };
6718            let mut valid = true;
6719            for family in &source.families {
6720                let owns = match &family.members {
6721                    crate::engine::SourceFamilyMembers::CompleteDomain(domain) => {
6722                        contains(domain.rect(), coord)
6723                    }
6724                    crate::engine::SourceFamilyMembers::ExplicitMembers(members) => {
6725                        members.as_slice().binary_search(&coord).is_ok()
6726                    }
6727                };
6728                if owns && !agrees(family.source_id, true) {
6729                    valid = false;
6730                }
6731            }
6732            for family in &source.partitioned_families {
6733                if family
6734                    .fragments
6735                    .iter()
6736                    .any(|fragment| contains(fragment.rect(), coord))
6737                    && !agrees(family.source_id, true)
6738                {
6739                    valid = false;
6740                }
6741                if let Some(member) = family
6742                    .legacy_members
6743                    .as_slice()
6744                    .iter()
6745                    .find(|member| member.coord == coord)
6746                    && !agrees(
6747                        family.source_id,
6748                        member.kind == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember,
6749                    )
6750                {
6751                    valid = false;
6752                }
6753            }
6754            if !valid {
6755                return Err(ExcelError::new(ExcelErrorKind::Value)
6756                    .with_message("indexed source ownership mismatch"));
6757            }
6758        }
6759        prepared.selected_points = Some(selected_points);
6760        prepared.replay_records = replay_records;
6761        Ok(Some(prepared))
6762    }
6763
6764    fn prepare_target_source_package(
6765        &mut self,
6766        sheet: &str,
6767        lease: StagedPackageLease,
6768        deadline: Option<std::time::Instant>,
6769    ) -> Result<PreparedTargetSourcePackage, ExcelError> {
6770        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
6771            ExcelError::new(ExcelErrorKind::Ref)
6772                .with_message(format!("deferred source sheet not found: {sheet}"))
6773        })?;
6774        let (
6775            source_report,
6776            families,
6777            partitions,
6778            replay,
6779            invalidated,
6780            suppressed,
6781            consumed_members,
6782            consumed_engine,
6783            reconciliation_replay,
6784        ) = {
6785            let package = self
6786                .staged_formulas
6787                .get(sheet)
6788                .and_then(|staged| staged.deferred_package.as_ref())
6789                .ok_or_else(|| {
6790                    ExcelError::new(ExcelErrorKind::Value)
6791                        .with_message("staged deferred source package is unavailable")
6792                })?;
6793            if package.sheet_name != sheet {
6794                return Err(ExcelError::new(ExcelErrorKind::Value)
6795                    .with_message("deferred formula package sheet mismatch"));
6796            }
6797            (
6798                package.accounting_report(),
6799                package.families.clone(),
6800                package.partitioned_families.clone(),
6801                Arc::clone(&package.replay),
6802                package.invalidated.clone(),
6803                package.suppressed.clone(),
6804                package.consumed_members.clone(),
6805                package.consumed_engine.clone(),
6806                package.reconciliation_replay.clone(),
6807            )
6808        };
6809
6810        let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
6811        for partition in &partitions {
6812            replay_disposition
6813                .register_partition(partition, false)
6814                .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
6815        }
6816        replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
6817        replay_disposition
6818            .register_consumed_members(consumed_engine.as_ref(), consumed_members.iter().copied());
6819        if !replay_disposition.consumed_engine_matches(&self.source_formula_token) {
6820            return Err(ExcelError::new(ExcelErrorKind::Value)
6821                .with_message("ResidualConsumedEngineMismatch"));
6822        }
6823        self.target_preparation_checkpoint(deadline, 1)?;
6824        let mut replay_records = if let Some(mut records) = reconciliation_replay {
6825            records.retain(|record| {
6826                let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
6827                else {
6828                    return true;
6829                };
6830                let coord = crate::engine::SourceCoord { row, col };
6831                let disposition = record.family.map_or_else(
6832                    || replay_disposition.ordinary_disposition(coord).0,
6833                    |family| replay_disposition.shared_disposition(family, coord),
6834                );
6835                !matches!(
6836                    disposition,
6837                    crate::engine::FormulaReplayCoordinateDisposition::Direct
6838                        | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
6839                )
6840            });
6841            records
6842        } else {
6843            replay
6844                .lock()
6845                .map_err(|_| {
6846                    ExcelError::new(ExcelErrorKind::Value)
6847                        .with_message("deferred formula spool lock poisoned")
6848                })?
6849                .replay_partitioned(&replay_disposition, &partitions)
6850                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
6851        };
6852        replay_records.sort_by_key(|record| record.source_order);
6853        for chunk in replay_records.chunks(256) {
6854            self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
6855        }
6856        if replay_records
6857            .windows(2)
6858            .any(|records| records[0].source_order == records[1].source_order)
6859        {
6860            return Err(ExcelError::new(ExcelErrorKind::Value)
6861                .with_message("duplicate deferred source-order proof"));
6862        }
6863
6864        let disposition = replay_disposition;
6865        let direct_families = 0usize;
6866        let direct_cells = 0u64;
6867        let direct_fragments = 0u64;
6868        let direct_complete_families = 0u64;
6869        let direct_complete_cells = 0u64;
6870        let direct_partition_families = 0u64;
6871        let direct_partition_cells = 0u64;
6872        let anchor_parses = 0u64;
6873        let anchor_asts = 0u64;
6874        let anchor_analyses = 0u64;
6875
6876        Ok(PreparedTargetSourcePackage {
6877            sheet: sheet.to_string(),
6878            sheet_id,
6879            lease,
6880            selected_points: None,
6881            complete_selections: Default::default(),
6882            deferred_shared: false,
6883            direct_domains: Vec::new(),
6884            source_report,
6885            replay_records,
6886            spool_replays: 1,
6887            disposition,
6888            legacy: Vec::new(),
6889            direct_families,
6890            direct_cells,
6891            direct_fragments,
6892            direct_complete_families,
6893            direct_complete_cells,
6894            direct_partition_families,
6895            direct_partition_cells,
6896            anchor_parses,
6897            anchor_asts,
6898            anchor_analyses,
6899        })
6900    }
6901
6902    fn fallback_planning_snapshot(
6903        &self,
6904        batch: &FormulaIngestBatch,
6905    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
6906        let mut requests = Vec::new();
6907        for formula in &batch.formulas {
6908            let ast = self
6909                .graph
6910                .data_store()
6911                .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
6912                .ok_or_else(|| {
6913                    ExcelError::new(ExcelErrorKind::Value)
6914                        .with_message("ordered fallback AST is unavailable")
6915                })?;
6916            Self::collect_planning_function_requests(&ast, &mut requests);
6917        }
6918        requests.sort();
6919        requests.dedup();
6920        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6921            &self.resolver,
6922            requests,
6923        )
6924        .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
6925    }
6926
6927    fn prepare_legacy_batch_fallback(
6928        &mut self,
6929        batch: FormulaIngestBatch,
6930        function_provider: &dyn crate::traits::FunctionProvider,
6931    ) -> Result<
6932        (
6933            crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
6934            u64,
6935        ),
6936        ExcelError,
6937    > {
6938        let formula_count = batch.formulas.len() as u64;
6939        let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6940        let mut planned = Vec::with_capacity(batch.formulas.len());
6941        for record in batch.formulas {
6942            let placement = CellRef::new(
6943                sheet_id,
6944                Coord::from_excel(record.row, record.col, true, true),
6945            );
6946            let ingested = self
6947                .graph
6948                .ingest_pipeline(function_provider)
6949                .enable_function_semantics()
6950                .ingest_formula(
6951                    FormulaAstInput::RawArena(record.ast_id),
6952                    placement,
6953                    record.formula_text,
6954                )
6955                .map_err(|error| {
6956                    ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
6957                })?;
6958            planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
6959        }
6960        let plan = self
6961            .graph
6962            .prepare_legacy_graph_plan(sheet_id, planned)
6963            .map_err(|error| {
6964                ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
6965            })?;
6966        Ok((plan, formula_count))
6967    }
6968
6969    fn publish_compressed_partial_report(
6970        &mut self,
6971        report: &FormulaIngestReport,
6972        direct_report: &FormulaIngestReport,
6973    ) {
6974        if direct_report.source_family_promoted == 0
6975            && direct_report.graph_formula_cells_materialized == 0
6976        {
6977            return;
6978        }
6979        let mut published = report.clone();
6980        published.accumulate(direct_report);
6981        self.record_formula_ingest_report(published);
6982    }
6983
6984    fn finish_compressed_formula_sources(
6985        &mut self,
6986        batches: Vec<(
6987            FormulaIngestBatch,
6988            crate::engine::FormulaCompressedSourceReport,
6989            crate::engine::FormulaCompressedPreparation,
6990        )>,
6991    ) -> Result<FormulaIngestReport, ExcelError> {
6992        self.observe_function_semantic_epoch()?;
6993        if batches.iter().any(|(_, _, preparation)| {
6994            !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
6995        }) {
6996            return Err(ExcelError::new(ExcelErrorKind::Value)
6997                .with_message("compressed source preparation belongs to another engine"));
6998        }
6999        if batches.iter().any(|(fallback, _, preparation)| {
7000            preparation.sheet_name.as_ref() != fallback.sheet_name
7001        }) {
7002            return Err(ExcelError::new(ExcelErrorKind::Value)
7003                .with_message("compressed source preparation sheet mismatch"));
7004        }
7005        let initial_guard = crate::function_registry::semantic_epoch_read_guard();
7006        let initial_provider_revision = self.resolver.planning_semantic_revision();
7007        let mut fallback_batches = Vec::with_capacity(batches.len());
7008        let mut stale_fallback_batches = Vec::new();
7009        let mut pending_preparations = Vec::new();
7010        for (mut fallback, mut source, mut preparation) in batches {
7011            for formula in fallback.formulas.drain(..) {
7012                let source_order = formula.source_order.ok_or_else(|| {
7013                    ExcelError::new(ExcelErrorKind::Value).with_message(
7014                        "compressed source supplied formulas without source-order proof",
7015                    )
7016                })?;
7017                let text = formula.formula_text.ok_or_else(|| {
7018                    ExcelError::new(ExcelErrorKind::Value).with_message(
7019                        "ordered compressed fallback formula has no exact source text",
7020                    )
7021                })?;
7022                preparation
7023                    .eager_replay
7024                    .push(crate::engine::DeferredReplayFormula {
7025                        source_order,
7026                        row: formula.row,
7027                        col: formula.col,
7028                        text: text.to_string(),
7029                        family: formula.source_family,
7030                        partition_owner: formula.partition_owner,
7031                    });
7032            }
7033            preparation
7034                .eager_replay
7035                .sort_by_key(|record| record.source_order);
7036            source.source_spool_replays = source
7037                .source_spool_replays
7038                .saturating_add(preparation.preparation_spool_replays);
7039            let stale_reason = preparation
7040                .function_semantics_used
7041                .then(|| {
7042                    if preparation.function_provider_revision != initial_provider_revision {
7043                        Some("FunctionProviderRevisionChanged")
7044                    } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
7045                    {
7046                        Some("FunctionSemanticEpochChanged")
7047                    } else {
7048                        None
7049                    }
7050                })
7051                .flatten();
7052            let stale_semantics = stale_reason.is_some();
7053            for reason in preparation.rejected.values() {
7054                *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
7055            }
7056            // No family is ever placed directly: every family replays.
7057            source.replay_families = source.families_seen;
7058            source.replay_cells = source.family_cells_seen;
7059            let compressed = crate::engine::FormulaCompressedSourceBatch::new(
7060                fallback.sheet_name.clone(),
7061                source,
7062            );
7063            if stale_semantics {
7064                stale_fallback_batches.push((fallback, compressed));
7065            } else {
7066                fallback_batches.push((fallback, compressed));
7067            }
7068            pending_preparations.push((preparation, stale_semantics));
7069        }
7070        drop(initial_guard);
7071
7072        // Build known fallback graphs first. Stale batches are forced through legacy ingest.
7073        let configured_mode = self.config.formula_plane_mode;
7074        self.config.formula_plane_mode = FormulaPlaneMode::Off;
7075        let stale_result =
7076            self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
7077        self.config.formula_plane_mode = configured_mode;
7078        let mut report = stale_result?;
7079        report.mode = configured_mode;
7080
7081        self.config.formula_plane_mode = FormulaPlaneMode::Off;
7082        let fallback_result =
7083            self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
7084        self.config.formula_plane_mode = configured_mode;
7085        match fallback_result {
7086            Ok(fallback_report) => report.accumulate(&fallback_report),
7087            Err(error) => {
7088                self.record_formula_ingest_report(report);
7089                return Err(error);
7090            }
7091        }
7092
7093        let mut direct_report =
7094            FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
7095        for (preparation, _) in &pending_preparations {
7096            direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
7097                preparation.clean_rejected_anchor_counts[0]
7098                    .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
7099            );
7100            direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
7101                preparation.clean_rejected_anchor_counts[1]
7102                    .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
7103            );
7104            direct_report.source_anchor_analyses =
7105                direct_report.source_anchor_analyses.saturating_add(
7106                    preparation.clean_rejected_anchor_counts[2]
7107                        .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
7108                );
7109        }
7110        loop {
7111            #[cfg(any(test, feature = "test-support"))]
7112            if let Some(hook) = self.before_prepared_span_commit_hook.take() {
7113                hook();
7114            }
7115            let commit_guard = crate::function_registry::semantic_epoch_read_guard();
7116            let commit_provider_revision = self.resolver.planning_semantic_revision();
7117            let mut newly_stale = Vec::new();
7118            let mut current = Vec::new();
7119            for pending in pending_preparations.drain(..) {
7120                let stale_reason = pending
7121                    .0
7122                    .function_semantics_used
7123                    .then(|| {
7124                        if pending.0.function_provider_revision != commit_provider_revision {
7125                            Some("FunctionProviderRevisionChanged")
7126                        } else if self
7127                            .prepared_function_semantics_changed(&pending.0, &commit_guard)
7128                        {
7129                            Some("FunctionSemanticEpochChanged")
7130                        } else {
7131                            None
7132                        }
7133                    })
7134                    .flatten();
7135                if let Some(reason) = stale_reason {
7136                    newly_stale.push((pending, reason));
7137                } else {
7138                    current.push(pending);
7139                }
7140            }
7141            if !newly_stale.is_empty() {
7142                drop(commit_guard);
7143                for ((mut preparation, was_initially_stale), reason) in newly_stale {
7144                    preparation
7145                        .eager_replay
7146                        .sort_by_key(|record| record.source_order);
7147                    direct_report.source_spool_replays =
7148                        direct_report.source_spool_replays.saturating_add(1);
7149                    if !was_initially_stale {
7150                        direct_report
7151                            .fallback_reasons
7152                            .entry(reason.to_string())
7153                            .or_default();
7154                    }
7155                    preparation.function_semantics_used = false;
7156                    if !preparation.eager_replay.is_empty() {
7157                        current.push((preparation, false));
7158                    }
7159                }
7160                pending_preparations = current;
7161                continue;
7162            }
7163            drop(commit_guard);
7164
7165            // Replay fallback families in source order.
7166            enum SourceProposal {
7167                KnownFallback {
7168                    source_order: crate::engine::SourceFormulaOrder,
7169                    records: Vec<crate::engine::DeferredReplayFormula>,
7170                },
7171            }
7172
7173            impl SourceProposal {
7174                fn source_order(&self) -> crate::engine::SourceFormulaOrder {
7175                    match self {
7176                        Self::KnownFallback { source_order, .. } => *source_order,
7177                    }
7178                }
7179            }
7180
7181            for (mut preparation, _) in current {
7182                let mut proposals = Vec::with_capacity(preparation.eager_replay.len());
7183                let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
7184                for record in preparation.eager_replay.drain(..) {
7185                    if let Some(owner) = record.partition_owner.or(record.family) {
7186                        family_fallbacks.entry(owner).or_default().push(record);
7187                    } else {
7188                        proposals.push(SourceProposal::KnownFallback {
7189                            source_order: record.source_order,
7190                            records: vec![record],
7191                        });
7192                    }
7193                }
7194                for (_, mut records) in family_fallbacks {
7195                    records.sort_by_key(|record| record.source_order);
7196                    if records
7197                        .windows(2)
7198                        .any(|window| window[0].source_order == window[1].source_order)
7199                    {
7200                        self.publish_compressed_partial_report(&report, &direct_report);
7201                        return Err(ExcelError::new(ExcelErrorKind::Value)
7202                            .with_message("duplicate exact-replay source-order proof"));
7203                    }
7204                    let Some(source_order) = records.first().map(|record| record.source_order)
7205                    else {
7206                        self.publish_compressed_partial_report(&report, &direct_report);
7207                        return Err(ExcelError::new(ExcelErrorKind::Value)
7208                            .with_message("empty exact-replay fallback family"));
7209                    };
7210                    proposals.push(SourceProposal::KnownFallback {
7211                        source_order,
7212                        records,
7213                    });
7214                }
7215                proposals.sort_by_key(SourceProposal::source_order);
7216                if proposals
7217                    .windows(2)
7218                    .any(|window| window[0].source_order() == window[1].source_order())
7219                {
7220                    self.publish_compressed_partial_report(&report, &direct_report);
7221                    return Err(ExcelError::new(ExcelErrorKind::Value)
7222                        .with_message("ambiguous compressed source-order proof"));
7223                }
7224
7225                for proposal in proposals {
7226                    match proposal {
7227                        SourceProposal::KnownFallback { records, .. } => {
7228                            let batch = match self.formula_batch_from_exact_replay(
7229                                preparation.sheet_name.as_ref(),
7230                                records,
7231                            ) {
7232                                Ok(batch) => batch,
7233                                Err(error) => {
7234                                    self.publish_compressed_partial_report(&report, &direct_report);
7235                                    return Err(error);
7236                                }
7237                            };
7238                            if batch.is_empty() {
7239                                continue;
7240                            }
7241                            let snapshot = match self.fallback_planning_snapshot(&batch) {
7242                                Ok(snapshot) => snapshot,
7243                                Err(error) => {
7244                                    self.publish_compressed_partial_report(&report, &direct_report);
7245                                    return Err(error);
7246                                }
7247                            };
7248                            let commit_guard =
7249                                crate::function_registry::semantic_epoch_read_guard();
7250                            let provider_revision_initial =
7251                                self.resolver.planning_semantic_revision();
7252                            if (commit_guard.epoch() != snapshot.epoch()
7253                                && snapshot.semantic_changes_affect_requests_since_guarded(
7254                                    &commit_guard,
7255                                    snapshot.epoch(),
7256                                ))
7257                                || snapshot.provider_revision().is_some_and(|revision| {
7258                                    Some(revision) != provider_revision_initial
7259                                })
7260                            {
7261                                drop(commit_guard);
7262                                self.publish_compressed_partial_report(&report, &direct_report);
7263                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7264                                    "ordered fallback planning snapshot became stale",
7265                                ));
7266                            }
7267                            let (plan, formula_count) = match self
7268                                .prepare_legacy_batch_fallback(batch, &snapshot)
7269                            {
7270                                Ok(plan) => plan,
7271                                Err(error) => {
7272                                    drop(commit_guard);
7273                                    self.publish_compressed_partial_report(&report, &direct_report);
7274                                    return Err(error);
7275                                }
7276                            };
7277                            let provider_revision_after =
7278                                self.resolver.planning_semantic_revision();
7279                            if provider_revision_after != provider_revision_initial {
7280                                drop(commit_guard);
7281                                self.publish_compressed_partial_report(&report, &direct_report);
7282                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7283                                    "function provider changed while preparing ordered fallback",
7284                                ));
7285                            }
7286                            let graph_vertices = plan.new_vertex_count();
7287                            let Some(graph_edges) = plan.planned_edge_count() else {
7288                                drop(commit_guard);
7289                                self.publish_compressed_partial_report(&report, &direct_report);
7290                                return Err(ExcelError::new(ExcelErrorKind::Value)
7291                                    .with_message("prepared ordered fallback size overflow"));
7292                            };
7293                            if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
7294                            {
7295                                drop(commit_guard);
7296                                self.publish_compressed_partial_report(&report, &direct_report);
7297                                return Err(error);
7298                            }
7299                            if let Err(error) =
7300                                self.graph.validate_prepared_legacy_graph_plan(&plan)
7301                            {
7302                                drop(commit_guard);
7303                                self.publish_compressed_partial_report(&report, &direct_report);
7304                                return Err(ExcelError::new(ExcelErrorKind::Value)
7305                                    .with_message(error.to_string()));
7306                            }
7307                            #[cfg(test)]
7308                            if let Some(hook) = self
7309                                .before_legacy_fallback_final_provider_sample_hook
7310                                .take()
7311                            {
7312                                hook();
7313                            }
7314                            let provider_revision_final =
7315                                self.resolver.planning_semantic_revision();
7316                            if provider_revision_final != provider_revision_initial {
7317                                drop(commit_guard);
7318                                self.publish_compressed_partial_report(&report, &direct_report);
7319                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7320                                    "function provider changed after ordered fallback validation",
7321                                ));
7322                            }
7323                            let graph_formulas =
7324                                self.graph.apply_prevalidated_legacy_graph_plan(plan);
7325                            direct_report.formula_cells_seen = direct_report
7326                                .formula_cells_seen
7327                                .saturating_add(formula_count);
7328                            direct_report.graph_formula_cells_materialized = direct_report
7329                                .graph_formula_cells_materialized
7330                                .saturating_add(graph_formulas as u64);
7331                            direct_report.graph_vertices_created = direct_report
7332                                .graph_vertices_created
7333                                .saturating_add(graph_vertices as u64);
7334                            direct_report.graph_edges_created = direct_report
7335                                .graph_edges_created
7336                                .saturating_add(graph_edges as u64);
7337                        }
7338                    }
7339                    #[cfg(test)]
7340                    if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
7341                        hook();
7342                    }
7343                }
7344            }
7345            break;
7346        }
7347        report.accumulate(&direct_report);
7348        self.record_formula_ingest_report(report.clone());
7349        Ok(report)
7350    }
7351    /// Ingest replayed per-cell formulas while preserving compressed source counters.
7352    pub(crate) fn ingest_compressed_formula_source_batches(
7353        &mut self,
7354        batches: Vec<(
7355            FormulaIngestBatch,
7356            crate::engine::FormulaCompressedSourceBatch,
7357        )>,
7358    ) -> Result<FormulaIngestReport, ExcelError> {
7359        self.ingest_compressed_formula_source_batches_inner(batches, true)
7360    }
7361
7362    fn ingest_compressed_formula_source_batches_inner(
7363        &mut self,
7364        batches: Vec<(
7365            FormulaIngestBatch,
7366            crate::engine::FormulaCompressedSourceBatch,
7367        )>,
7368        publish_report: bool,
7369    ) -> Result<FormulaIngestReport, ExcelError> {
7370        let mut source_counts = [0_u64; 11];
7371        let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
7372        let mut formula_batches = Vec::with_capacity(batches.len());
7373        let mut compressed_families = Vec::new();
7374        let mut partitioned_families = Vec::new();
7375        for (batch, compressed_batch) in batches {
7376            let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
7377            if sheet_name.as_ref() != batch.sheet_name {
7378                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7379                    "compressed formula source sheet does not match its replay batch",
7380                ));
7381            }
7382            compressed_families.push((batch.sheet_name.clone(), families));
7383            partitioned_families.push((batch.sheet_name.clone(), partitions));
7384            source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
7385            source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
7386            source_counts[2] =
7387                source_counts[2].saturating_add(compressed.source_shared_anchor_events);
7388            source_counts[3] =
7389                source_counts[3].saturating_add(compressed.source_shared_descendant_events);
7390            source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
7391            source_counts[5] =
7392                source_counts[5].saturating_add(compressed.source_formula_records_spooled);
7393            source_counts[6] =
7394                source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
7395            source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
7396            source_counts[8] =
7397                source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
7398            source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
7399            source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
7400            source_report.families_seen = source_report
7401                .families_seen
7402                .saturating_add(compressed.families_seen);
7403            source_report.family_cells_seen = source_report
7404                .family_cells_seen
7405                .saturating_add(compressed.family_cells_seen);
7406            source_report.source_clean_families = source_report
7407                .source_clean_families
7408                .saturating_add(compressed.source_clean_families);
7409            source_report.source_clean_cells = source_report
7410                .source_clean_cells
7411                .saturating_add(compressed.source_clean_cells);
7412            source_report.source_fragmentable_families = source_report
7413                .source_fragmentable_families
7414                .saturating_add(compressed.source_fragmentable_families);
7415            source_report.source_fragmentable_cells = source_report
7416                .source_fragmentable_cells
7417                .saturating_add(compressed.source_fragmentable_cells);
7418            source_report.source_fragment_count = source_report
7419                .source_fragment_count
7420                .saturating_add(compressed.source_fragment_count);
7421            source_report.source_isolated_fallback_cells = source_report
7422                .source_isolated_fallback_cells
7423                .saturating_add(compressed.source_isolated_fallback_cells);
7424            source_report.source_hole_exclusions = source_report
7425                .source_hole_exclusions
7426                .saturating_add(compressed.source_hole_exclusions);
7427            source_report.source_ordinary_exclusions = source_report
7428                .source_ordinary_exclusions
7429                .saturating_add(compressed.source_ordinary_exclusions);
7430            source_report.source_partition_failures = source_report
7431                .source_partition_failures
7432                .saturating_add(compressed.source_partition_failures);
7433            source_report.replay_families = source_report
7434                .replay_families
7435                .saturating_add(compressed.replay_families);
7436            source_report.replay_cells = source_report
7437                .replay_cells
7438                .saturating_add(compressed.replay_cells);
7439            source_report.forward_descendants = source_report
7440                .forward_descendants
7441                .saturating_add(compressed.forward_descendants);
7442            source_report.evidence_limit_fallbacks = source_report
7443                .evidence_limit_fallbacks
7444                .saturating_add(compressed.evidence_limit_fallbacks);
7445            source_report.evidence_peak_bytes = source_report
7446                .evidence_peak_bytes
7447                .max(compressed.evidence_peak_bytes);
7448            for (reason, count) in compressed.fallback_reasons {
7449                *source_report.fallback_reasons.entry(reason).or_default() += count;
7450            }
7451            formula_batches.push(batch);
7452        }
7453        self.ingest_formula_batches_inner(
7454            formula_batches,
7455            source_counts,
7456            Some(source_report),
7457            compressed_families,
7458            partitioned_families,
7459            publish_report,
7460        )
7461    }
7462
7463    pub fn ingest_formula_batches(
7464        &mut self,
7465        batches: Vec<FormulaIngestBatch>,
7466    ) -> Result<FormulaIngestReport, ExcelError> {
7467        let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
7468        let report = self.ingest_formula_batches_inner(
7469            batches,
7470            [0; 11],
7471            None,
7472            Vec::new(),
7473            Vec::new(),
7474            true,
7475        )?;
7476        if has_formulas {
7477            self.mark_topology_edited();
7478        }
7479        Ok(report)
7480    }
7481
7482    fn ingest_formula_batches_unpublished(
7483        &mut self,
7484        batches: Vec<FormulaIngestBatch>,
7485    ) -> Result<FormulaIngestReport, ExcelError> {
7486        self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
7487    }
7488
7489    fn ingest_formula_batches_inner(
7490        &mut self,
7491        batches: Vec<FormulaIngestBatch>,
7492        source_counts: [u64; 11],
7493        source_report: Option<crate::engine::FormulaCompressedSourceReport>,
7494        compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
7495        partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
7496        publish_report: bool,
7497    ) -> Result<FormulaIngestReport, ExcelError> {
7498        self.observe_function_semantic_epoch()?;
7499        let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7500        #[cfg(feature = "tracing")]
7501        let arena_nodes_before = self.graph.data_store().memory_usage().total_ast_nodes;
7502        #[cfg(feature = "tracing")]
7503        let route = if !partitioned_families.is_empty() {
7504            "partitioned"
7505        } else if !compressed_families.is_empty() {
7506            "compressed_source"
7507        } else if source_report.is_some() {
7508            "replay"
7509        } else {
7510            "ordinary"
7511        };
7512        let _ingest_span = crate::engine::trace::fz_span!(
7513            tracing::Level::INFO,
7514            "ingest",
7515            "ingest.batch",
7516            mode = ?FormulaPlaneMode::Off,
7517            route,
7518            formula_cells = formula_cells_seen
7519        );
7520        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
7521        let materialize_batches = batches;
7522        report.formula_cells_seen = formula_cells_seen;
7523        report.source_formula_events = source_counts[0];
7524        report.source_ordinary_events = source_counts[1];
7525        report.source_shared_anchor_events = source_counts[2];
7526        report.source_shared_descendant_events = source_counts[3];
7527        report.source_unknown_events = source_counts[4];
7528        report.source_formula_records_spooled = source_counts[5];
7529        report.source_spool_encoded_bytes = source_counts[6];
7530        report.source_spool_peak_memory_bytes = source_counts[7];
7531        report.source_spool_spilled_bytes = source_counts[8];
7532        report.source_spool_spill_files = source_counts[9];
7533        report.source_spool_replays = source_counts[10];
7534        if let Some(source) = source_report {
7535            report.source_families_seen = source.families_seen;
7536            report.source_family_cells_seen = source.family_cells_seen;
7537            report.source_family_shadow_eligible = source.source_clean_families;
7538            report.source_family_shadow_eligible_cells = source.source_clean_cells;
7539            report.source_partitioned_families_seen = source.source_fragmentable_families;
7540            report.source_partition_holes = source.source_hole_exclusions;
7541            report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
7542            report.source_partition_failures = source.source_partition_failures;
7543            report.source_partition_surviving_cells = source.source_fragmentable_cells;
7544            report.source_family_fallback = report
7545                .source_family_fallback
7546                .saturating_add(source.replay_families);
7547            report.source_family_fallback_cells = report
7548                .source_family_fallback_cells
7549                .saturating_add(source.replay_cells);
7550            report.source_forward_descendants = source.forward_descendants;
7551            report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
7552            report.source_evidence_peak_bytes = source.evidence_peak_bytes;
7553            for (reason, count) in source.fallback_reasons {
7554                let total = report.fallback_reasons.entry(reason).or_default();
7555                *total = total.saturating_add(count);
7556            }
7557        }
7558
7559        // A first load without graph admission plans in the builder, one
7560        // chunk at a time (a load error fails the load, so the builder's
7561        // partial application on a planning error is unobservable).
7562        if self.graph.first_load_assume_new()
7563            && !self.graph_admission_enabled()
7564            && !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
7565        {
7566            let mut builder =
7567                crate::engine::ingest_builder::BulkIngestBuilder::new(&mut self.graph);
7568            for batch in materialize_batches {
7569                if batch.is_empty() {
7570                    continue;
7571                }
7572                let sheet_id = builder
7573                    .add_sheet_checked(&batch.sheet_name)
7574                    .ok_or_else(|| {
7575                        ExcelError::new(ExcelErrorKind::Ref)
7576                            .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7577                    })?;
7578                builder.add_formula_refs(
7579                    sheet_id,
7580                    batch.formulas.into_iter().map(|record| {
7581                        (
7582                            record.row,
7583                            record.col,
7584                            crate::engine::graph::FormulaRef::of_ingested(
7585                                record.ast_id,
7586                                record.member_anchor,
7587                            ),
7588                        )
7589                    }),
7590                );
7591            }
7592            let summary = builder.finish_with_provider(&self.resolver)?;
7593            report.graph_formula_cells_materialized = summary.formulas as u64;
7594            report.graph_vertices_created = summary.vertices as u64;
7595            report.graph_edges_created = summary.edges as u64;
7596        } else if !materialize_batches.iter().all(FormulaIngestBatch::is_empty) {
7597            let mut prepared_by_sheet: BTreeMap<String, Vec<_>> = BTreeMap::new();
7598            for batch in materialize_batches {
7599                if batch.is_empty() {
7600                    continue;
7601                }
7602                let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
7603                    ExcelError::new(ExcelErrorKind::Ref)
7604                        .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7605                })?;
7606                let mut pipeline = self.ingest_pipeline();
7607                // Plan each record and keep only what the graph needs (the
7608                // pipeline's per-formula facts are dropped at once).
7609                let prepared = prepared_by_sheet.entry(batch.sheet_name).or_default();
7610                prepared.reserve(batch.formulas.len());
7611                for record in batch.formulas {
7612                    let placement = CellRef::new(
7613                        sheet_id,
7614                        Coord::from_excel(record.row, record.col, true, true),
7615                    );
7616                    let input = match record.member_anchor {
7617                        Some(anchor) => FormulaAstInput::Member {
7618                            template: record.ast_id,
7619                            anchor,
7620                        },
7621                        None => FormulaAstInput::RawArena(record.ast_id),
7622                    };
7623                    let formula = pipeline.ingest_formula(input, placement, None)?;
7624                    prepared.push((
7625                        record.row,
7626                        record.col,
7627                        crate::engine::graph::FormulaRef::of_ingested(
7628                            formula.ast_id,
7629                            formula.member_anchor,
7630                        ),
7631                        formula.dep_plan,
7632                    ));
7633                }
7634            }
7635            let admission_preflighted = self.graph_admission_enabled();
7636            if admission_preflighted {
7637                let mut preview = Vec::new();
7638                for (sheet_name, formulas) in &prepared_by_sheet {
7639                    let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
7640                        ExcelError::new(ExcelErrorKind::Ref)
7641                            .with_message(format!("unknown ingest sheet: {sheet_name}"))
7642                    })?;
7643                    preview.extend(
7644                        formulas
7645                            .iter()
7646                            .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
7647                    );
7648                }
7649                let admission = self.graph.preview_formula_mutations(&preview)?;
7650                self.preflight_graph_admission(admission)?;
7651            }
7652
7653            let mut builder = self.begin_bulk_ingest();
7654            if admission_preflighted {
7655                builder.mark_admission_preflighted();
7656            }
7657            for (sheet_name, formulas) in prepared_by_sheet {
7658                if formulas.is_empty() {
7659                    continue;
7660                }
7661                let sheet_id = builder.add_sheet(&sheet_name);
7662                builder.add_formula_plans(sheet_id, formulas);
7663            }
7664            let summary = builder.finish()?;
7665            report.graph_formula_cells_materialized = summary.formulas as u64;
7666            report.graph_vertices_created = summary.vertices as u64;
7667            report.graph_edges_created = summary.edges as u64;
7668        }
7669
7670        crate::engine::trace::fz_event!(
7671            tracing::Level::INFO,
7672            "ingest",
7673            "ingest.summary",
7674            candidate_cells = report.shadow_candidate_cells,
7675            accepted_span_cells = report.shadow_accepted_span_cells,
7676            fallback_cells = report.shadow_fallback_cells,
7677            spans_created = report.shadow_spans_created,
7678            templates_interned = report.shadow_templates_interned,
7679            graph_vertices_created = report.graph_vertices_created,
7680            graph_edges_created = report.graph_edges_created,
7681            arena_nodes_delta = self
7682                .graph
7683                .data_store()
7684                .memory_usage()
7685                .total_ast_nodes
7686                .saturating_sub(arena_nodes_before)
7687        );
7688        if publish_report {
7689            self.record_formula_ingest_report(report.clone());
7690        }
7691        Ok(report)
7692    }
7693
7694    fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
7695        let mut unique = Vec::new();
7696        for diagnostic in self.formula_parse_diagnostics.drain(start..) {
7697            let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
7698                prior.sheet == diagnostic.sheet
7699                    && prior.row == diagnostic.row
7700                    && prior.col == diagnostic.col
7701                    && prior.formula == diagnostic.formula
7702                    && prior.policy == diagnostic.policy
7703            });
7704            if !duplicate {
7705                unique.push(diagnostic);
7706            }
7707        }
7708        self.formula_parse_diagnostics.extend(unique);
7709    }
7710
7711    pub fn handle_formula_parse_error(
7712        &mut self,
7713        sheet: &str,
7714        row: u32,
7715        col: u32,
7716        formula: &str,
7717        message: String,
7718    ) -> Result<Option<ASTNode>, ExcelError> {
7719        let policy = self.config.formula_parse_policy;
7720
7721        if policy == FormulaParsePolicy::Strict {
7722            let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
7723            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7724                "Formula parse error at {sheet}!{col_a1}{row}: {message}"
7725            )));
7726        }
7727
7728        self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
7729            sheet: sheet.to_string(),
7730            row,
7731            col,
7732            formula: formula.to_string(),
7733            message: message.clone(),
7734            policy,
7735        });
7736
7737        match policy {
7738            FormulaParsePolicy::Strict => unreachable!(),
7739            FormulaParsePolicy::KeepCachedValue => Ok(None),
7740            FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
7741                ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
7742                None,
7743            ))),
7744            FormulaParsePolicy::CoerceToError => {
7745                let err = ExcelError::new(ExcelErrorKind::Error)
7746                    .with_message(format!("Malformed formula: {message}"));
7747                Ok(Some(ASTNode::new(
7748                    ASTNodeType::Literal(LiteralValue::Error(err)),
7749                    None,
7750                )))
7751            }
7752        }
7753    }
7754
7755    #[cfg(test)]
7756    fn target_preparation_fault(
7757        &mut self,
7758        seam: crate::engine::target_preparation::TargetPreparationFault,
7759    ) -> Result<(), ExcelError> {
7760        if self.target_preparation_fault_for_test == Some(seam) {
7761            self.target_preparation_fault_for_test = None;
7762            Err(ExcelError::new(ExcelErrorKind::Value)
7763                .with_message(format!("injected target preparation fault: {seam:?}")))
7764        } else {
7765            Ok(())
7766        }
7767    }
7768
7769    fn preparation_stale(
7770        reason: formualizer_common::PreparationStaleReason,
7771        message: impl Into<String>,
7772    ) -> ExcelError {
7773        ExcelError::new(ExcelErrorKind::Value)
7774            .with_message(message)
7775            .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
7776    }
7777
7778    fn preparation_revision_stale_reason(
7779        assumptions: &crate::engine::PreparationRevision,
7780        current: &crate::engine::PreparationRevision,
7781        planning_requests: &BTreeSet<(String, String, usize)>,
7782        staged_leases_match: bool,
7783    ) -> Option<formualizer_common::PreparationStaleReason> {
7784        if assumptions.graph != current.graph {
7785            Some(formualizer_common::PreparationStaleReason::Graph)
7786        } else if assumptions.staged != current.staged || !staged_leases_match {
7787            Some(formualizer_common::PreparationStaleReason::Staged)
7788        } else if assumptions.symbols != current.symbols {
7789            Some(formualizer_common::PreparationStaleReason::Symbols)
7790        } else if assumptions.provider != current.provider {
7791            Some(formualizer_common::PreparationStaleReason::Provider)
7792        } else if assumptions.semantic != current.semantic
7793            && crate::function_registry::semantic_changes_affect_requests_since(
7794                assumptions.semantic,
7795                planning_requests.iter().cloned(),
7796            )
7797        {
7798            Some(formualizer_common::PreparationStaleReason::Semantic)
7799        } else {
7800            None
7801        }
7802    }
7803
7804    fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
7805        crate::engine::PreparationRevision {
7806            graph: self.graph.topology_revision(),
7807            authority: 0,
7808            authority_indexes: 0,
7809            authority_indexed_plane: 0,
7810            staged: self.staged_formula_index.revision(),
7811            symbols: self.graph.symbol_revision(),
7812            semantic: crate::function_registry::semantic_epoch(),
7813            provider: self.resolver.planning_semantic_revision(),
7814        }
7815    }
7816
7817    fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
7818        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
7819        let provider = self.resolver.planning_semantic_revision();
7820        let semantic = registry_guard.epoch();
7821        PlanningRevisionSnapshot {
7822            engine_topology_epoch: self.topology_epoch,
7823            graph_topology_revision: self.graph.topology_revision(),
7824            staged: self.staged_formula_index.revision(),
7825            symbols: self.graph.symbol_revision(),
7826            semantic,
7827            provider,
7828            deterministic_mode: self.config.deterministic_mode.clone(),
7829            budgets: self.evaluation_resource_budgets.clone(),
7830        }
7831    }
7832
7833    fn recalc_plan_key(&self) -> RecalcPlanKey {
7834        RecalcPlanKey {
7835            engine_token: Arc::clone(&self.recalc_plan_token),
7836            revisions: self.planning_revision_snapshot(),
7837        }
7838    }
7839
7840    fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
7841        ExcelError::new(ExcelErrorKind::Value)
7842            .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
7843            .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
7844    }
7845
7846    fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
7847        use formualizer_common::PlanStaleReason;
7848
7849        if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
7850            return Err(Self::plan_stale(PlanStaleReason::Engine));
7851        }
7852
7853        let current = self.planning_revision_snapshot();
7854        let expected = &key.revisions;
7855        let stale = if expected.provider != current.provider {
7856            Some(PlanStaleReason::Provider)
7857        } else if expected.semantic != current.semantic {
7858            Some(PlanStaleReason::Semantic)
7859        } else if expected.budgets != current.budgets
7860            || expected.deterministic_mode != current.deterministic_mode
7861        {
7862            Some(PlanStaleReason::Budget)
7863        } else if expected.staged != current.staged {
7864            Some(PlanStaleReason::Staged)
7865        } else if expected.symbols != current.symbols {
7866            Some(PlanStaleReason::Symbols)
7867        } else if expected.graph_topology_revision != current.graph_topology_revision
7868            || expected.engine_topology_epoch != current.engine_topology_epoch
7869        {
7870            Some(PlanStaleReason::Graph)
7871        } else {
7872            None
7873        };
7874        stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
7875    }
7876
7877    fn target_preparation_checkpoint(
7878        &mut self,
7879        deadline: Option<std::time::Instant>,
7880        work: u64,
7881    ) -> Result<(), ExcelError> {
7882        if self
7883            .active_cancel_flag
7884            .as_ref()
7885            .is_some_and(|cancel| cancel.is_cancelled())
7886        {
7887            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
7888                .with_message("target graph preparation cancelled"));
7889        }
7890        if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
7891            return Err(crate::engine::ResourceLedgerError::Exhausted(
7892                formualizer_common::ResourceExhaustionDetail {
7893                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
7894                    limit: 0,
7895                    observed: 1,
7896                    request_id: self
7897                        .active_evaluation_resource_request
7898                        .as_ref()
7899                        .map(|stats| stats.request_id),
7900                },
7901            )
7902            .into_excel_error());
7903        }
7904        self.charge_bounded_work(work)
7905    }
7906
7907    fn opaque_reason_in_ast(
7908        &self,
7909        ast: &ASTNode,
7910        provider: &crate::function_registry::RegistryPlanningSnapshot,
7911    ) -> Option<crate::engine::OpaqueReason> {
7912        match &ast.node_type {
7913            ASTNodeType::Function { name, args } => {
7914                let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
7915                if canonical == "INDIRECT" {
7916                    return Some(crate::engine::OpaqueReason::RuntimeTextReference);
7917                }
7918                let Some(caps) = provider.function_capabilities("", name) else {
7919                    return Some(crate::engine::OpaqueReason::UnknownFunction);
7920                };
7921                if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
7922                    || caps.contains(FnCaps::RETURNS_REFERENCE)
7923                {
7924                    return Some(crate::engine::OpaqueReason::DynamicReference);
7925                }
7926                args.iter()
7927                    .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
7928            }
7929            ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
7930            ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
7931            ASTNodeType::BinaryOp { left, right, .. } => self
7932                .opaque_reason_in_ast(left, provider)
7933                .or_else(|| self.opaque_reason_in_ast(right, provider)),
7934            ASTNodeType::Array(rows) => rows
7935                .iter()
7936                .flat_map(|row| row.iter())
7937                .find_map(|item| self.opaque_reason_in_ast(item, provider)),
7938            ASTNodeType::Reference {
7939                reference:
7940                    ReferenceType::Cell {
7941                        sheet: Some(sheet), ..
7942                    }
7943                    | ReferenceType::Range {
7944                        sheet: Some(sheet), ..
7945                    },
7946                ..
7947            } if self.graph.sheet_id(sheet).is_none() => {
7948                Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
7949            }
7950            ASTNodeType::Reference {
7951                reference:
7952                    ReferenceType::External(_)
7953                    | ReferenceType::Cell3D { .. }
7954                    | ReferenceType::Range3D { .. },
7955                ..
7956            } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
7957            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
7958        }
7959    }
7960
7961    fn target_planning_snapshot(
7962        &mut self,
7963        ast: &ASTNode,
7964        planning_requests: &mut BTreeSet<(String, String, usize)>,
7965    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7966        #[cfg(test)]
7967        if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
7968            return Err(Self::preparation_stale(
7969                formualizer_common::PreparationStaleReason::Semantic,
7970                "injected target semantic preparation movement",
7971            ));
7972        }
7973        #[cfg(test)]
7974        if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
7975            hook();
7976        }
7977        let mut requests = Vec::new();
7978        Self::collect_planning_function_requests(ast, &mut requests);
7979        requests.sort();
7980        requests.dedup();
7981        planning_requests.extend(requests.iter().cloned());
7982        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7983            &self.resolver,
7984            requests,
7985        )
7986        .map_err(|error| {
7987            ExcelError::new(ExcelErrorKind::Value)
7988                .with_message(format!("target planning snapshot unavailable: {error:?}"))
7989        })
7990    }
7991
7992    fn target_planning_snapshot_stale_reason(
7993        snapshot: &crate::function_registry::RegistryPlanningSnapshot,
7994        assumptions: &crate::engine::PreparationRevision,
7995    ) -> Option<formualizer_common::PreparationStaleReason> {
7996        if snapshot
7997            .provider_revision()
7998            .is_some_and(|revision| Some(revision) != assumptions.provider)
7999        {
8000            Some(formualizer_common::PreparationStaleReason::Provider)
8001        } else if snapshot.epoch() != assumptions.semantic
8002            && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
8003        {
8004            Some(formualizer_common::PreparationStaleReason::Semantic)
8005        } else {
8006            None
8007        }
8008    }
8009
8010    fn widen_target_preparation(
8011        policy: crate::engine::OpaquePreparePolicy,
8012        scope: &mut crate::engine::PrepareScope,
8013        reasons: &mut Vec<crate::engine::OpaqueReason>,
8014        reason: crate::engine::OpaqueReason,
8015    ) -> Result<bool, ExcelError> {
8016        if policy == crate::engine::OpaquePreparePolicy::Error {
8017            return Err(ExcelError::new(ExcelErrorKind::NImpl)
8018                .with_message(format!("opaque target preparation semantics: {reason:?}")));
8019        }
8020        if !reasons.contains(&reason) {
8021            reasons.push(reason);
8022        }
8023        if !matches!(scope, crate::engine::PrepareScope::Workbook) {
8024            *scope = crate::engine::PrepareScope::Workbook;
8025            Ok(true)
8026        } else {
8027            Ok(false)
8028        }
8029    }
8030
8031    fn widen_target_preparation_to_sheet(
8032        policy: crate::engine::OpaquePreparePolicy,
8033        scope: &mut crate::engine::PrepareScope,
8034        reasons: &mut Vec<crate::engine::OpaqueReason>,
8035        reason: crate::engine::OpaqueReason,
8036        sheet: &str,
8037    ) -> Result<bool, ExcelError> {
8038        if policy == crate::engine::OpaquePreparePolicy::Error {
8039            return Err(ExcelError::new(ExcelErrorKind::NImpl)
8040                .with_message(format!("opaque target preparation semantics: {reason:?}")));
8041        }
8042        if !reasons.contains(&reason) {
8043            reasons.push(reason);
8044        }
8045        match scope {
8046            crate::engine::PrepareScope::Exact => {
8047                *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
8048                Ok(true)
8049            }
8050            crate::engine::PrepareScope::Sheets(sheets) => {
8051                if sheets.iter().any(|candidate| candidate == sheet) {
8052                    Ok(false)
8053                } else {
8054                    sheets.push(sheet.to_string());
8055                    sheets.sort();
8056                    Ok(true)
8057                }
8058            }
8059            crate::engine::PrepareScope::Workbook => Ok(false),
8060        }
8061    }
8062
8063    fn ast_has_proven_sheet_local_dynamic(
8064        ast: &ASTNode,
8065        provider: &crate::function_registry::RegistryPlanningSnapshot,
8066    ) -> bool {
8067        fn classify(
8068            ast: &ASTNode,
8069            provider: &crate::function_registry::RegistryPlanningSnapshot,
8070        ) -> (bool, bool) {
8071            match &ast.node_type {
8072                ASTNodeType::Function { name, args } => {
8073                    let Some(caps) = provider.function_capabilities("", name) else {
8074                        return (false, false);
8075                    };
8076                    let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
8077                        || caps.contains(FnCaps::RETURNS_REFERENCE);
8078                    let canonical = name.rsplit('.').next().unwrap_or(name);
8079                    if dynamic
8080                        && !canonical.eq_ignore_ascii_case("OFFSET")
8081                        && !canonical.eq_ignore_ascii_case("INDEX")
8082                    {
8083                        return (false, true);
8084                    }
8085                    let mut has_dynamic = dynamic;
8086                    for arg in args {
8087                        let (safe, child_dynamic) = classify(arg, provider);
8088                        if !safe {
8089                            return (false, has_dynamic || child_dynamic);
8090                        }
8091                        has_dynamic |= child_dynamic;
8092                    }
8093                    (true, has_dynamic)
8094                }
8095                ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
8096                ASTNodeType::BinaryOp { left, right, .. } => {
8097                    let (left_safe, left_dynamic) = classify(left, provider);
8098                    let (right_safe, right_dynamic) = classify(right, provider);
8099                    (left_safe && right_safe, left_dynamic || right_dynamic)
8100                }
8101                ASTNodeType::Array(rows) => {
8102                    let mut has_dynamic = false;
8103                    for item in rows.iter().flatten() {
8104                        let (safe, child_dynamic) = classify(item, provider);
8105                        if !safe {
8106                            return (false, has_dynamic || child_dynamic);
8107                        }
8108                        has_dynamic |= child_dynamic;
8109                    }
8110                    (true, has_dynamic)
8111                }
8112                ASTNodeType::Reference {
8113                    reference:
8114                        ReferenceType::Cell { sheet: None, .. }
8115                        | ReferenceType::Range { sheet: None, .. },
8116                    ..
8117                }
8118                | ASTNodeType::Literal(_)
8119                | ASTNodeType::Omitted => (true, false),
8120                ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
8121            }
8122        }
8123
8124        let (safe, dynamic) = classify(ast, provider);
8125        safe && dynamic
8126    }
8127
8128    fn table_selection_region(
8129        &self,
8130        entry: &crate::engine::graph::TableEntry,
8131        selection: &crate::engine::TableSelection,
8132    ) -> Result<PreparationRegion, ExcelError> {
8133        let mut start_row = entry.range.start.coord.row() + 1;
8134        let mut end_row = entry.range.end.coord.row() + 1;
8135        let mut start_col = entry.range.start.coord.col() + 1;
8136        let mut end_col = entry.range.end.coord.col() + 1;
8137        match selection {
8138            crate::engine::TableSelection::Whole => {}
8139            crate::engine::TableSelection::Headers => {
8140                if !entry.header_row {
8141                    return Err(ExcelError::new(ExcelErrorKind::Value)
8142                        .with_message(format!("table {} has no header row", entry.name)));
8143                }
8144                end_row = start_row;
8145            }
8146            crate::engine::TableSelection::Data => {
8147                if entry.header_row {
8148                    start_row = start_row.saturating_add(1);
8149                }
8150                if entry.totals_row {
8151                    end_row = end_row.saturating_sub(1);
8152                }
8153            }
8154            crate::engine::TableSelection::Totals => {
8155                if !entry.totals_row {
8156                    return Err(ExcelError::new(ExcelErrorKind::Value)
8157                        .with_message(format!("table {} has no totals row", entry.name)));
8158                }
8159                start_row = end_row;
8160            }
8161            crate::engine::TableSelection::Column(column) => {
8162                let index = entry.col_index(column).ok_or_else(|| {
8163                    ExcelError::new(ExcelErrorKind::Name)
8164                        .with_message(format!("unknown table column: {column}"))
8165                })?;
8166                start_col = start_col.saturating_add(index as u32);
8167                end_col = start_col;
8168            }
8169            crate::engine::TableSelection::Columns { start, end } => {
8170                let first = entry.col_index(start).ok_or_else(|| {
8171                    ExcelError::new(ExcelErrorKind::Name)
8172                        .with_message(format!("unknown table column: {start}"))
8173                })?;
8174                let last = entry.col_index(end).ok_or_else(|| {
8175                    ExcelError::new(ExcelErrorKind::Name)
8176                        .with_message(format!("unknown table column: {end}"))
8177                })?;
8178                if first > last {
8179                    return Err(ExcelError::new(ExcelErrorKind::Value)
8180                        .with_message("table column selection is reversed"));
8181                }
8182                start_col = start_col.saturating_add(first as u32);
8183                end_col = entry
8184                    .range
8185                    .start
8186                    .coord
8187                    .col()
8188                    .saturating_add(last as u32)
8189                    .saturating_add(1);
8190            }
8191        }
8192        if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
8193            start_row = entry.range.start.coord.row() + 1;
8194            end_row = start_row;
8195        }
8196        if start_row > end_row || start_col > end_col {
8197            return Err(
8198                ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
8199            );
8200        }
8201        Ok(PreparationRegion {
8202            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
8203            sheet_id: entry.sheet_id(),
8204            start_row,
8205            start_col,
8206            end_row,
8207            end_col,
8208        })
8209    }
8210
8211    #[cfg(test)]
8212    pub(crate) fn reset_target_root_dedup_probes_for_test() {
8213        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
8214    }
8215
8216    #[cfg(test)]
8217    pub(crate) fn target_root_dedup_probes_for_test() -> usize {
8218        TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
8219    }
8220
8221    pub(crate) fn resolve_target_producers(
8222        &mut self,
8223        targets: &[crate::engine::EvaluationTarget],
8224    ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
8225        use crate::engine::target_preparation::TargetProducer;
8226
8227        let request_id = self
8228            .active_evaluation_resource_request
8229            .as_ref()
8230            .map(|request| request.request_id);
8231        let mut roots = OrderedTargetProducers::with_capacity(targets.len())
8232            .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
8233        let resolve_region = |engine: &mut Self,
8234                              region: Region,
8235                              value_only: Option<CellRef>,
8236                              roots: &mut OrderedTargetProducers|
8237         -> Result<(), ExcelError> {
8238            let before = roots.len();
8239            for anchor in engine.graph.spill_anchors_in_region(
8240                region.sheet_id(),
8241                region.axis_ranges().0.query_bounds().0,
8242                region.axis_ranges().1.query_bounds().0,
8243                region.axis_ranges().0.query_bounds().1,
8244                region.axis_ranges().1.query_bounds().1,
8245            ) {
8246                roots
8247                    .push(TargetProducer::Legacy(anchor))
8248                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8249            }
8250            for vertex in engine.graph.vertices_in_region(
8251                region.sheet_id(),
8252                region.axis_ranges().0.query_bounds().0,
8253                region.axis_ranges().0.query_bounds().1,
8254                region.axis_ranges().1.query_bounds().0,
8255                region.axis_ranges().1.query_bounds().1,
8256            ) {
8257                let vertex = engine
8258                    .graph
8259                    .get_cell_ref(vertex)
8260                    .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
8261                    .unwrap_or(vertex);
8262                // `vertices_in_region` is a sheet-index query and a sheet index holds only
8263                // grid-addressed vertices, so a region can never yield a symbol: names,
8264                // tables and external sources have no position for a region to cover.
8265                // Symbol roots come from the by-name lookups below instead.
8266                if matches!(
8267                    engine.graph.get_vertex_kind(vertex),
8268                    VertexKind::FormulaScalar | VertexKind::FormulaArray
8269                ) {
8270                    roots
8271                        .push(TargetProducer::Legacy(vertex))
8272                        .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8273                }
8274            }
8275            if roots.len() == before
8276                && let Some(cell) = value_only
8277            {
8278                roots
8279                    .push(TargetProducer::ValueOnly(cell))
8280                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8281            }
8282            Ok(())
8283        };
8284
8285        for target in targets {
8286            match target {
8287                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8288                    if *row == 0 || *col == 0 {
8289                        return Err(ExcelError::new(ExcelErrorKind::Ref)
8290                            .with_message("target cell coordinates are one-based"));
8291                    }
8292                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8293                        ExcelError::new(ExcelErrorKind::Ref)
8294                            .with_message(format!("target sheet not found: {sheet}"))
8295                    })?;
8296                    let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
8297                    resolve_region(
8298                        self,
8299                        Region::point(sheet_id, *row - 1, *col - 1),
8300                        Some(cell),
8301                        &mut roots,
8302                    )?;
8303                }
8304                crate::engine::EvaluationTarget::Range(range) => {
8305                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8306                        ExcelError::new(ExcelErrorKind::Ref)
8307                            .with_message(format!("target sheet not found: {}", range.sheet))
8308                    })?;
8309                    resolve_region(
8310                        self,
8311                        Region::rect(
8312                            sheet_id,
8313                            range.start_row - 1,
8314                            range.end_row - 1,
8315                            range.start_col - 1,
8316                            range.end_col - 1,
8317                        ),
8318                        None,
8319                        &mut roots,
8320                    )?;
8321                }
8322                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8323                    let scope = self.name_query_scope(scope_sheet.as_deref())?;
8324                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
8325                        roots
8326                            .push(TargetProducer::Symbol(entry.vertex))
8327                            .map_err(|_| {
8328                                target_root_allocation_error(roots.len() + 1, request_id)
8329                            })?;
8330                    }
8331                }
8332                crate::engine::EvaluationTarget::Table { name, .. } => {
8333                    if let Some(entry) = self.graph.resolve_table_entry(name) {
8334                        roots
8335                            .push(TargetProducer::Symbol(entry.vertex))
8336                            .map_err(|_| {
8337                                target_root_allocation_error(roots.len() + 1, request_id)
8338                            })?;
8339                    }
8340                }
8341            }
8342        }
8343        Ok(roots.into_vec())
8344    }
8345
8346    /// Transactionally prepare the complete ordinary staged demand closure for typed targets.
8347    ///
8348    /// This method prepares graph topology only. It does not evaluate target values.
8349    pub fn prepare_graph_for_targets(
8350        &mut self,
8351        targets: &[crate::engine::EvaluationTarget],
8352        options: crate::engine::TargetEvalOptions<'_>,
8353    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8354        let previous_budgets = self.evaluation_resource_budgets.clone();
8355        let diagnostics_len = self.formula_parse_diagnostics.len();
8356        let previous_report = self.last_formula_ingest_report.clone();
8357        if let Some(budgets) = options.budgets {
8358            self.evaluation_resource_budgets = budgets.clone();
8359        }
8360        let previous_graph_budget_override = self
8361            .graph
8362            .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
8363        // Hoist the call's cancellation onto the engine for its duration, so the
8364        // preparation checkpoints observe it. This is a standalone entry point, so
8365        // the previous value is restored rather than cleared.
8366        let previous_cancel = self.active_cancel_flag.take();
8367        self.active_cancel_flag = options.cancel.clone();
8368        let result = self.observe_evaluation_resource_request(
8369            EvaluationRequestKind::TargetPreparation,
8370            |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
8371        );
8372        self.active_cancel_flag = previous_cancel;
8373        self.graph
8374            .set_admission_budget_override(previous_graph_budget_override);
8375        self.evaluation_resource_budgets = previous_budgets;
8376        if result.is_err() {
8377            self.formula_parse_diagnostics.truncate(diagnostics_len);
8378            self.last_formula_ingest_report = previous_report;
8379        }
8380        result
8381    }
8382
8383    fn prepare_graph_for_targets_unobserved(
8384        &mut self,
8385        targets: &[crate::engine::EvaluationTarget],
8386        options: &crate::engine::TargetEvalOptions<'_>,
8387    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8388        let scratch_checkpoint = self
8389            .active_resource_ledger
8390            .as_ref()
8391            .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
8392        let result = self.prepare_graph_for_targets_transaction(targets, options);
8393        let release = self
8394            .active_resource_ledger
8395            .as_mut()
8396            .map_or(Ok(()), |ledger| {
8397                ledger.release_scratch_to(scratch_checkpoint)
8398            });
8399        match (result, release) {
8400            (result, Ok(())) => result,
8401            (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
8402        }
8403    }
8404
8405    fn prepare_graph_for_targets_transaction(
8406        &mut self,
8407        targets: &[crate::engine::EvaluationTarget],
8408        options: &crate::engine::TargetEvalOptions<'_>,
8409    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8410        use crate::engine::{
8411            OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
8412            TableSelection,
8413        };
8414
8415        self.target_preparation_checkpoint(options.deadline, 0)?;
8416        self.observe_function_semantic_epoch()?;
8417        let assumptions = self.preparation_revisions();
8418        let ledger_at_start = self
8419            .active_resource_ledger
8420            .as_ref()
8421            .map(|ledger| ledger.snapshot());
8422        let diagnostics_len = self.formula_parse_diagnostics.len();
8423        let report_len = self.last_formula_ingest_report.clone();
8424        let request_id = options.request_id.or_else(|| {
8425            self.active_evaluation_resource_request
8426                .as_ref()
8427                .map(|stats| stats.request_id)
8428        });
8429
8430        let mut scope = PrepareScope::Exact;
8431        let mut reasons = Vec::new();
8432        let mut regions = VecDeque::new();
8433        let mut deferred_shared_regions = VecDeque::new();
8434        let mut normalized = Vec::with_capacity(targets.len());
8435        let mut symbol_vertices = VecDeque::new();
8436        for target in targets {
8437            self.target_preparation_checkpoint(options.deadline, 1)?;
8438            match target {
8439                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8440                    if *row == 0 || *col == 0 {
8441                        return Err(ExcelError::new(ExcelErrorKind::Ref)
8442                            .with_message("target cell coordinates are one-based"));
8443                    }
8444                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8445                        ExcelError::new(ExcelErrorKind::Ref)
8446                            .with_message(format!("target sheet not found: {sheet}"))
8447                    })?;
8448                    regions.push_back(PreparationRegion {
8449                        sheet: sheet.clone(),
8450                        sheet_id,
8451                        start_row: *row,
8452                        start_col: *col,
8453                        end_row: *row,
8454                        end_col: *col,
8455                    });
8456                    normalized.push(target.clone());
8457                }
8458                crate::engine::EvaluationTarget::Range(range) => {
8459                    if range.start_row == 0
8460                        || range.start_col == 0
8461                        || range.end_row < range.start_row
8462                        || range.end_col < range.start_col
8463                    {
8464                        return Err(ExcelError::new(ExcelErrorKind::Ref)
8465                            .with_message("invalid target range"));
8466                    }
8467                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8468                        ExcelError::new(ExcelErrorKind::Ref)
8469                            .with_message(format!("target sheet not found: {}", range.sheet))
8470                    })?;
8471                    regions.push_back(PreparationRegion {
8472                        sheet: range.sheet.clone(),
8473                        sheet_id,
8474                        start_row: range.start_row,
8475                        start_col: range.start_col,
8476                        end_row: range.end_row,
8477                        end_col: range.end_col,
8478                    });
8479                    normalized.push(target.clone());
8480                }
8481                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8482                    let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
8483                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
8484                        symbol_vertices.push_back(entry.vertex);
8485                    } else {
8486                        Self::widen_target_preparation(
8487                            options.opaque_policy,
8488                            &mut scope,
8489                            &mut reasons,
8490                            OpaqueReason::UnresolvedName,
8491                        )?;
8492                    }
8493                    normalized.push(target.clone());
8494                }
8495                crate::engine::EvaluationTarget::Table { name, selection } => {
8496                    if let Some(entry) = self.graph.resolve_table_entry(name) {
8497                        let region = self.table_selection_region(entry, selection)?;
8498                        symbol_vertices.push_back(entry.vertex);
8499                        regions.push_back(region);
8500                    } else {
8501                        Self::widen_target_preparation(
8502                            options.opaque_policy,
8503                            &mut scope,
8504                            &mut reasons,
8505                            OpaqueReason::UnresolvedTable,
8506                        )?;
8507                    }
8508                    normalized.push(target.clone());
8509                }
8510            }
8511        }
8512
8513        let mut visited_regions = FxHashSet::default();
8514        let mut visited_vertices = FxHashSet::default();
8515        let mut selected = FxHashSet::default();
8516        let mut prepared = Vec::new();
8517        let mut pending_diagnostics = Vec::new();
8518        let mut planning_requests = BTreeSet::new();
8519        let mut selected_cells = Vec::new();
8520        let mut workbook_seeded = false;
8521        let mut sheet_scope_seeded = BTreeSet::new();
8522        let mut indexed_query_sheets = FxHashSet::default();
8523        let mut discovery_scratch_reserved = 0u64;
8524        let mut package_encountered = false;
8525        let mut selected_package_sheets = FxHashSet::default();
8526        let mut selected_package_points: BTreeMap<String, BTreeSet<(u32, u32)>> = BTreeMap::new();
8527        let mut prepared_packages: Vec<PreparedTargetSourcePackage> = Vec::new();
8528        let authoritative_with_ordinary = false;
8529        let has_unknown_package_sheet = self
8530            .staged_formula_index
8531            .package_sheets()
8532            .any(|sheet| self.graph.sheet_id(sheet).is_none());
8533
8534        loop {
8535            if let PrepareScope::Sheets(sheets) = &scope {
8536                for sheet in sheets.clone() {
8537                    if !sheet_scope_seeded.insert(sheet.clone()) {
8538                        continue;
8539                    }
8540                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8541                        package_encountered = true;
8542                        Self::widen_target_preparation(
8543                            options.opaque_policy,
8544                            &mut scope,
8545                            &mut reasons,
8546                            OpaqueReason::UnsupportedSourceSemantics,
8547                        )?;
8548                        continue;
8549                    };
8550                    for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
8551                        self.target_preparation_checkpoint(options.deadline, 1)?;
8552                        regions.push_back(PreparationRegion {
8553                            sheet: sheet.clone(),
8554                            sheet_id,
8555                            start_row: lease.row,
8556                            start_col: lease.col,
8557                            end_row: lease.row,
8558                            end_col: lease.col,
8559                        });
8560                    }
8561                    if self
8562                        .staged_formula_index
8563                        .package_lease_for_sheet(&sheet)
8564                        .is_some()
8565                    {
8566                        regions.push_back(PreparationRegion {
8567                            sheet: sheet.clone(),
8568                            sheet_id,
8569                            start_row: 1,
8570                            start_col: 1,
8571                            end_row: self.workbook_load_limits.max_sheet_rows,
8572                            end_col: self.workbook_load_limits.max_sheet_cols,
8573                        });
8574                    }
8575                }
8576            }
8577            if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
8578                workbook_seeded = true;
8579                for (sheet, lease) in self.staged_formula_index.all_leases() {
8580                    self.target_preparation_checkpoint(options.deadline, 1)?;
8581                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8582                        package_encountered = true;
8583                        Self::widen_target_preparation(
8584                            options.opaque_policy,
8585                            &mut scope,
8586                            &mut reasons,
8587                            OpaqueReason::UnsupportedSourceSemantics,
8588                        )?;
8589                        continue;
8590                    };
8591                    regions.push_back(PreparationRegion {
8592                        sheet,
8593                        sheet_id,
8594                        start_row: lease.row,
8595                        start_col: lease.col,
8596                        end_row: lease.row,
8597                        end_col: lease.col,
8598                    });
8599                }
8600                let package_sheets = self
8601                    .staged_formula_index
8602                    .package_sheets()
8603                    .map(str::to_string)
8604                    .collect::<Vec<_>>();
8605                for sheet in package_sheets {
8606                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8607                        package_encountered = true;
8608                        Self::widen_target_preparation(
8609                            options.opaque_policy,
8610                            &mut scope,
8611                            &mut reasons,
8612                            OpaqueReason::UnsupportedSourceSemantics,
8613                        )?;
8614                        continue;
8615                    };
8616                    regions.push_back(PreparationRegion {
8617                        sheet,
8618                        sheet_id,
8619                        start_row: 1,
8620                        start_col: 1,
8621                        end_row: self.workbook_load_limits.max_sheet_rows,
8622                        end_col: self.workbook_load_limits.max_sheet_cols,
8623                    });
8624                }
8625            }
8626
8627            // Finish known ordinary/name dependency discovery before expanding
8628            // partial shared demands. A queued SUM may complete those families.
8629            let allow_partial_shared = regions.is_empty() && symbol_vertices.is_empty();
8630            let next_region = if allow_partial_shared {
8631                deferred_shared_regions.pop_front()
8632            } else {
8633                regions.pop_front()
8634            };
8635            let Some(region) = next_region else {
8636                if let Some(vertex) = symbol_vertices.pop_front() {
8637                    self.target_preparation_checkpoint(options.deadline, 1)?;
8638                    if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
8639                        continue;
8640                    }
8641                    let vertex_is_dynamic = self.graph.is_dynamic(vertex);
8642                    if let Some(ast) = self.graph.get_formula(vertex) {
8643                        let snapshot =
8644                            self.target_planning_snapshot(&ast, &mut planning_requests)?;
8645                        if let Some(reason) =
8646                            Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
8647                        {
8648                            return Err(Self::preparation_stale(
8649                                reason,
8650                                "target planning snapshot became stale during discovery",
8651                            ));
8652                        }
8653                        let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
8654                        if let Some(reason) =
8655                            opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
8656                        {
8657                            if reason == OpaqueReason::DynamicReference
8658                                && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
8659                            {
8660                                let sheet = self.graph.get_vertex_sheet_id(vertex);
8661                                let sheet = self.graph.sheet_name(sheet).to_string();
8662                                Self::widen_target_preparation_to_sheet(
8663                                    options.opaque_policy,
8664                                    &mut scope,
8665                                    &mut reasons,
8666                                    reason,
8667                                    &sheet,
8668                                )?;
8669                            } else {
8670                                Self::widen_target_preparation(
8671                                    options.opaque_policy,
8672                                    &mut scope,
8673                                    &mut reasons,
8674                                    reason,
8675                                )?;
8676                            }
8677                        }
8678                    } else if vertex_is_dynamic {
8679                        Self::widen_target_preparation(
8680                            options.opaque_policy,
8681                            &mut scope,
8682                            &mut reasons,
8683                            OpaqueReason::DynamicReference,
8684                        )?;
8685                    }
8686                    if let Some(anchor) = self
8687                        .graph
8688                        .get_cell_ref(vertex)
8689                        .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
8690                    {
8691                        symbol_vertices.push_back(anchor);
8692                    }
8693                    if let Some(cell) = self.graph.get_cell_ref(vertex) {
8694                        let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
8695                        regions.push_back(PreparationRegion {
8696                            sheet,
8697                            sheet_id: cell.sheet_id,
8698                            start_row: cell.coord.row() + 1,
8699                            start_col: cell.coord.col() + 1,
8700                            end_row: cell.coord.row() + 1,
8701                            end_col: cell.coord.col() + 1,
8702                        });
8703                    }
8704                    // The formula's direct precedents, from the authority:
8705                    // cells and ranges become regions, symbol rows (names,
8706                    // tables, sources) their vertices.
8707                    match self.graph.authority_vertex_precedents(vertex) {
8708                        Some(precedents) => {
8709                            for (sheet_id, rect) in precedents {
8710                                self.target_preparation_checkpoint(options.deadline, 1)?;
8711                                if sheet_id == crate::engine::authority::geom::SYMBOL_SHEET {
8712                                    for slot in rect.r0..=rect.r1 {
8713                                        if let Some(symbol) =
8714                                            self.graph.authority_host().symbols().vertex(slot)
8715                                        {
8716                                            symbol_vertices.push_back(symbol);
8717                                        }
8718                                    }
8719                                    continue;
8720                                }
8721                                let sheet = self.graph.sheet_name(sheet_id).to_string();
8722                                regions.push_back(PreparationRegion {
8723                                    sheet,
8724                                    sheet_id,
8725                                    start_row: rect.r0 + 1,
8726                                    start_col: rect.c0 + 1,
8727                                    end_row: (rect.r1 + 1)
8728                                        .min(self.workbook_load_limits.max_sheet_rows),
8729                                    end_col: (rect.c1 + 1)
8730                                        .min(self.workbook_load_limits.max_sheet_cols),
8731                                });
8732                            }
8733                        }
8734                        // The authority cannot answer (failed host): widen.
8735                        None => {
8736                            if Self::widen_target_preparation(
8737                                options.opaque_policy,
8738                                &mut scope,
8739                                &mut reasons,
8740                                OpaqueReason::UnresolvedCrossSheetBinding,
8741                            )? {
8742                                continue;
8743                            }
8744                        }
8745                    }
8746                    if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
8747                        match &name.definition {
8748                            NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
8749                                sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
8750                                sheet_id: cell.sheet_id,
8751                                start_row: cell.coord.row() + 1,
8752                                start_col: cell.coord.col() + 1,
8753                                end_row: cell.coord.row() + 1,
8754                                end_col: cell.coord.col() + 1,
8755                            }),
8756                            NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
8757                                sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
8758                                sheet_id: range.start.sheet_id,
8759                                start_row: range.start.coord.row() + 1,
8760                                start_col: range.start.coord.col() + 1,
8761                                end_row: range.end.coord.row() + 1,
8762                                end_col: range.end.coord.col() + 1,
8763                            }),
8764                            NamedDefinition::Formula {
8765                                ast,
8766                                dependencies,
8767                                range_deps,
8768                            } => {
8769                                let snapshot =
8770                                    self.target_planning_snapshot(ast, &mut planning_requests)?;
8771                                if let Some(reason) = Self::target_planning_snapshot_stale_reason(
8772                                    &snapshot,
8773                                    &assumptions,
8774                                ) {
8775                                    return Err(Self::preparation_stale(
8776                                        reason,
8777                                        "target planning snapshot became stale during discovery",
8778                                    ));
8779                                }
8780                                if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
8781                                    Self::widen_target_preparation(
8782                                        options.opaque_policy,
8783                                        &mut scope,
8784                                        &mut reasons,
8785                                        reason,
8786                                    )?;
8787                                }
8788                                for dependency in dependencies {
8789                                    self.target_preparation_checkpoint(options.deadline, 1)?;
8790                                    symbol_vertices.push_back(*dependency);
8791                                }
8792                                for range in range_deps {
8793                                    self.target_preparation_checkpoint(options.deadline, 1)?;
8794                                    // `Current` is the sheet this name's formula
8795                                    // was interpreted on, which is the sheet its
8796                                    // vertex is placed on -- the same derivation
8797                                    // the formula-vertex arm above uses. It is
8798                                    // never the workbook default sheet, and an
8799                                    // unresolvable `Name` widens instead of
8800                                    // silently landing on some other sheet.
8801                                    let context_sheet = self.graph.get_vertex_sheet_id(vertex);
8802                                    let Ok(sheet_id) =
8803                                        self.resolve_sheet_locator(&range.sheet, context_sheet)
8804                                    else {
8805                                        if Self::widen_target_preparation(
8806                                            options.opaque_policy,
8807                                            &mut scope,
8808                                            &mut reasons,
8809                                            OpaqueReason::UnresolvedCrossSheetBinding,
8810                                        )? {
8811                                            break;
8812                                        }
8813                                        continue;
8814                                    };
8815                                    regions.push_back(PreparationRegion {
8816                                        sheet: self.graph.sheet_name(sheet_id).to_string(),
8817                                        sheet_id,
8818                                        start_row: range
8819                                            .start_row
8820                                            .map_or(1, |bound| bound.index + 1),
8821                                        start_col: range
8822                                            .start_col
8823                                            .map_or(1, |bound| bound.index + 1),
8824                                        end_row: range.end_row.map_or(
8825                                            self.workbook_load_limits.max_sheet_rows,
8826                                            |bound| bound.index + 1,
8827                                        ),
8828                                        end_col: range.end_col.map_or(
8829                                            self.workbook_load_limits.max_sheet_cols,
8830                                            |bound| bound.index + 1,
8831                                        ),
8832                                    });
8833                                }
8834                            }
8835                            NamedDefinition::Literal(_) => {}
8836                        }
8837                    }
8838                    if let Some(table) = self.graph.table_by_vertex(vertex) {
8839                        regions
8840                            .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
8841                    }
8842                    continue;
8843                }
8844                break;
8845            };
8846
8847            self.target_preparation_checkpoint(options.deadline, 1)?;
8848            if !visited_regions.insert(region.clone()) && !allow_partial_shared {
8849                continue;
8850            }
8851            if let PrepareScope::Sheets(sheets) = &mut scope
8852                && !sheets.iter().any(|sheet| sheet == &region.sheet)
8853            {
8854                sheets.push(region.sheet.clone());
8855                sheets.sort();
8856            }
8857            let package_match = self.staged_formula_index.package_for_region(
8858                &region.sheet,
8859                region.start_row,
8860                region.start_col,
8861                region.end_row,
8862                region.end_col,
8863            );
8864            let package_lease = match package_match {
8865                Some(Ok(lease)) => Some(lease),
8866                Some(Err(()))
8867                    if region.start_row == 1
8868                        && region.start_col == 1
8869                        && region.end_row == self.workbook_load_limits.max_sheet_rows
8870                        && region.end_col == self.workbook_load_limits.max_sheet_cols =>
8871                {
8872                    self.staged_formula_index
8873                        .package_lease_for_sheet(&region.sheet)
8874                }
8875                Some(Err(())) => {
8876                    Self::widen_target_preparation(
8877                        options.opaque_policy,
8878                        &mut scope,
8879                        &mut reasons,
8880                        OpaqueReason::DeferredSourcePackage,
8881                    )?;
8882                    None
8883                }
8884                None => None,
8885            };
8886            let compatibility_before_package_replay = package_lease.is_some()
8887                && (authoritative_with_ordinary || has_unknown_package_sheet);
8888            let package_lease = if compatibility_before_package_replay {
8889                package_encountered = true;
8890                Self::widen_target_preparation(
8891                    options.opaque_policy,
8892                    &mut scope,
8893                    &mut reasons,
8894                    OpaqueReason::UnsupportedSourceSemantics,
8895                )?;
8896                None
8897            } else {
8898                package_lease
8899            };
8900            if let Some(package_lease) = package_lease
8901                && !selected_package_sheets.contains(&region.sheet)
8902            {
8903                self.target_preparation_checkpoint(options.deadline, 1)?;
8904                let mut points = self.staged_formula_index.package_points_in_region(
8905                    &region.sheet,
8906                    region.start_row,
8907                    region.start_col,
8908                    region.end_row,
8909                    region.end_col,
8910                );
8911                if let Some(selected) = selected_package_points.get(&region.sheet) {
8912                    points.retain(|point| !selected.contains(point));
8913                }
8914
8915                if let Some(selected) = selected_package_points.get(&region.sheet) {
8916                    points.retain(|point| !selected.contains(point));
8917                }
8918                let permit_partial = allow_partial_shared
8919                    || (points.len() == 1
8920                        && regions.is_empty()
8921                        && symbol_vertices.is_empty()
8922                        && deferred_shared_regions.is_empty());
8923                let partial = self.prepare_target_exact_source_selection(
8924                    &region.sheet,
8925                    package_lease,
8926                    points,
8927                    selected_package_points
8928                        .get(&region.sheet)
8929                        .unwrap_or(&BTreeSet::new()),
8930                    permit_partial,
8931                    options.deadline,
8932                    &mut discovery_scratch_reserved,
8933                )?;
8934                let mut package = if let Some(mut package) = partial {
8935                    if package.deferred_shared {
8936                        deferred_shared_regions.push_back(region.clone());
8937                    }
8938                    // A later range can complete a family touched earlier in this
8939                    // request. Retire only that family's earlier legacy proposals;
8940                    // nothing has been published and its dependencies remain demanded.
8941                    if !package.direct_domains.is_empty() {
8942                        for prior in prepared_packages
8943                            .iter_mut()
8944                            .filter(|prior| prior.sheet == region.sheet)
8945                        {
8946                            prior
8947                                .replay_records
8948                                .retain(|record| !package.direct_contains(record.row, record.col));
8949                            prior
8950                                .legacy
8951                                .retain(|(row, col, _, _)| !package.direct_contains(*row, *col));
8952                            if let Some(points) = prior.selected_points.as_mut() {
8953                                points.retain(|&(row, col)| !package.direct_contains(row, col));
8954                            }
8955                        }
8956                    }
8957                    if selected_package_points.contains_key(&region.sheet) {
8958                        package.source_report = Default::default();
8959                    }
8960                    let points = package.selected_points.as_ref().unwrap();
8961                    if !points.is_empty() {
8962                        selected_package_points
8963                            .entry(region.sheet.clone())
8964                            .or_default()
8965                            .extend(points.iter().copied());
8966                    }
8967                    package
8968                } else {
8969                    selected_package_sheets.insert(region.sheet.clone());
8970                    self.prepare_target_source_package(
8971                        &region.sheet,
8972                        package_lease,
8973                        options.deadline,
8974                    )?
8975                };
8976                if package
8977                    .selected_points
8978                    .as_ref()
8979                    .is_none_or(|points| !points.is_empty())
8980                {
8981                    let mut final_fallback = BTreeMap::new();
8982                    for record in package.fallback_records() {
8983                        final_fallback.insert((record.row, record.col), record.clone());
8984                    }
8985                    let batch = self.formula_batch_from_exact_replay(
8986                        &region.sheet,
8987                        final_fallback.into_values(),
8988                    )?;
8989                    for record in batch.formulas {
8990                        self.target_preparation_checkpoint(options.deadline, 1)?;
8991                        let ast = self
8992                            .graph
8993                            .data_store()
8994                            .retrieve_ast(record.ast_id, self.graph.sheet_reg())
8995                            .ok_or_else(|| {
8996                                ExcelError::new(ExcelErrorKind::Value)
8997                                    .with_message("target fallback AST is unavailable")
8998                            })?;
8999                        let snapshot =
9000                            self.target_planning_snapshot(&ast, &mut planning_requests)?;
9001                        if let Some(reason) =
9002                            Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9003                        {
9004                            return Err(Self::preparation_stale(
9005                                reason,
9006                                "target fallback planning snapshot became stale during discovery",
9007                            ));
9008                        }
9009                        let proven_sheet_local_dynamic =
9010                            Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9011                        if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9012                            if reason == OpaqueReason::DynamicReference
9013                                && proven_sheet_local_dynamic
9014                            {
9015                                Self::widen_target_preparation_to_sheet(
9016                                    options.opaque_policy,
9017                                    &mut scope,
9018                                    &mut reasons,
9019                                    reason,
9020                                    &region.sheet,
9021                                )?;
9022                            } else {
9023                                Self::widen_target_preparation(
9024                                    options.opaque_policy,
9025                                    &mut scope,
9026                                    &mut reasons,
9027                                    reason,
9028                                )?;
9029                            }
9030                        }
9031                        let placement = CellRef::new(
9032                            package.sheet_id,
9033                            Coord::from_excel(record.row, record.col, true, true),
9034                        );
9035                        let ingested = self
9036                            .graph
9037                            .ingest_pipeline(&snapshot)
9038                            .enable_function_semantics()
9039                            .ingest_formula(
9040                                FormulaAstInput::RawArena(record.ast_id),
9041                                placement,
9042                                record.formula_text,
9043                            )?;
9044                        if ingested.dep_plan.dynamic {
9045                            if proven_sheet_local_dynamic {
9046                                Self::widen_target_preparation_to_sheet(
9047                                    options.opaque_policy,
9048                                    &mut scope,
9049                                    &mut reasons,
9050                                    OpaqueReason::DynamicReference,
9051                                    &region.sheet,
9052                                )?;
9053                            } else {
9054                                Self::widen_target_preparation(
9055                                    options.opaque_policy,
9056                                    &mut scope,
9057                                    &mut reasons,
9058                                    OpaqueReason::DynamicReference,
9059                                )?;
9060                            }
9061                        }
9062                        for dep in &ingested.dep_plan.direct_cell_deps {
9063                            self.target_preparation_checkpoint(options.deadline, 1)?;
9064                            regions.push_back(PreparationRegion {
9065                                sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9066                                sheet_id: dep.sheet_id,
9067                                start_row: dep.coord.row().saturating_add(1),
9068                                start_col: dep.coord.col().saturating_add(1),
9069                                end_row: dep.coord.row().saturating_add(1),
9070                                end_col: dep.coord.col().saturating_add(1),
9071                            });
9072                        }
9073                        for range in &ingested.dep_plan.range_deps {
9074                            self.target_preparation_checkpoint(options.deadline, 1)?;
9075                            // `Current` is the sheet the staged package's formula
9076                            // lives on.
9077                            let Ok(dependency_sheet) =
9078                                self.resolve_sheet_locator(&range.sheet, package.sheet_id)
9079                            else {
9080                                Self::widen_target_preparation(
9081                                    options.opaque_policy,
9082                                    &mut scope,
9083                                    &mut reasons,
9084                                    OpaqueReason::UnresolvedCrossSheetBinding,
9085                                )?;
9086                                continue;
9087                            };
9088                            regions.push_back(PreparationRegion {
9089                                sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9090                                sheet_id: dependency_sheet,
9091                                start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9092                                start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9093                                end_row: range
9094                                    .end_row
9095                                    .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9096                                        bound.index + 1
9097                                    }),
9098                                end_col: range
9099                                    .end_col
9100                                    .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9101                                        bound.index + 1
9102                                    }),
9103                            });
9104                        }
9105                        for name in ingested
9106                            .dep_plan
9107                            .resolved_named_refs
9108                            .iter()
9109                            .chain(&ingested.dep_plan.named_refs)
9110                        {
9111                            self.target_preparation_checkpoint(options.deadline, 1)?;
9112                            if let Some(entry) =
9113                                self.graph.resolve_name_entry(name, package.sheet_id)
9114                            {
9115                                symbol_vertices.push_back(entry.vertex);
9116                            } else if self.graph.resolve_source_scalar_entry(name).is_none()
9117                                && self.graph.resolve_source_table_entry(name).is_none()
9118                            {
9119                                Self::widen_target_preparation(
9120                                    options.opaque_policy,
9121                                    &mut scope,
9122                                    &mut reasons,
9123                                    OpaqueReason::UnresolvedName,
9124                                )?;
9125                            }
9126                        }
9127                        for table in &ingested.dep_plan.table_refs {
9128                            self.target_preparation_checkpoint(options.deadline, 1)?;
9129                            if let Some(entry) = self.graph.resolve_table_entry(table) {
9130                                symbol_vertices.push_back(entry.vertex);
9131                            } else if self.graph.resolve_source_table_entry(table).is_none() {
9132                                Self::widen_target_preparation(
9133                                    options.opaque_policy,
9134                                    &mut scope,
9135                                    &mut reasons,
9136                                    OpaqueReason::UnresolvedTable,
9137                                )?;
9138                            }
9139                        }
9140                        package.legacy.push((
9141                            record.row,
9142                            record.col,
9143                            ingested.ast_id,
9144                            ingested.dep_plan,
9145                        ));
9146                    }
9147                    prepared_packages.push(package);
9148                }
9149            }
9150            let leases = self.staged_formula_index.leases_in_region(
9151                &region.sheet,
9152                region.start_row,
9153                region.start_col,
9154                region.end_row,
9155                region.end_col,
9156            );
9157            for lease in leases {
9158                self.target_preparation_checkpoint(options.deadline, 1)?;
9159                let sheet_id = self.graph.sheet_id(&region.sheet).ok_or_else(|| {
9160                    ExcelError::new(ExcelErrorKind::Ref)
9161                        .with_message(format!("staged formula sheet not found: {}", region.sheet))
9162                })?;
9163                let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
9164                if !selected.insert(key) {
9165                    continue;
9166                }
9167                let text = self
9168                    .staged_formulas
9169                    .get(&region.sheet)
9170                    .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
9171                    .ok_or_else(|| {
9172                        ExcelError::new(ExcelErrorKind::Value)
9173                            .with_message("staged formula index is stale")
9174                    })?
9175                    .to_string();
9176                let formula = if text.starts_with('=') {
9177                    text.clone()
9178                } else {
9179                    format!("={text}")
9180                };
9181                self.target_preparation_checkpoint(options.deadline, 1)?;
9182                let ast = match formualizer_parse::parser::parse(&formula) {
9183                    Ok(ast) => ast,
9184                    Err(error) => {
9185                        if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
9186                            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9187                                format!(
9188                                    "Formula parse error at {}!{}{}: {error}",
9189                                    region.sheet,
9190                                    col_letters_from_1based(lease.col)
9191                                        .unwrap_or_else(|_| "?".to_string()),
9192                                    lease.row
9193                                ),
9194                            ));
9195                        }
9196                        pending_diagnostics.push(FormulaParseDiagnostic {
9197                            sheet: region.sheet.clone(),
9198                            row: lease.row,
9199                            col: lease.col,
9200                            formula: formula.clone(),
9201                            message: error.to_string(),
9202                            policy: self.config.formula_parse_policy,
9203                        });
9204                        match self.config.formula_parse_policy {
9205                            FormulaParsePolicy::KeepCachedValue => {
9206                                selected_cells.push(
9207                                    formualizer_common::RangeAddress::new(
9208                                        region.sheet.clone(),
9209                                        lease.row,
9210                                        lease.col,
9211                                        lease.row,
9212                                        lease.col,
9213                                    )
9214                                    .expect("selected staged coordinates are valid"),
9215                                );
9216                                prepared.push(PreparedOrdinaryStagedFormula {
9217                                    sheet: region.sheet.clone(),
9218                                    sheet_id,
9219                                    lease,
9220                                    ast_id: None,
9221                                    plan: None,
9222                                });
9223                                continue;
9224                            }
9225                            FormulaParsePolicy::AsText => ASTNode::new(
9226                                ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
9227                                None,
9228                            ),
9229                            FormulaParsePolicy::CoerceToError => ASTNode::new(
9230                                ASTNodeType::Literal(LiteralValue::Error(
9231                                    ExcelError::new(ExcelErrorKind::Error)
9232                                        .with_message(format!("Malformed formula: {error}")),
9233                                )),
9234                                None,
9235                            ),
9236                            FormulaParsePolicy::Strict => unreachable!(),
9237                        }
9238                    }
9239                };
9240                self.target_preparation_checkpoint(options.deadline, 1)?;
9241                let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
9242                self.target_preparation_checkpoint(options.deadline, 1)?;
9243                if let Some(reason) =
9244                    Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9245                {
9246                    return Err(Self::preparation_stale(
9247                        reason,
9248                        "target planning snapshot became stale during discovery",
9249                    ));
9250                }
9251                if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9252                    if reason == OpaqueReason::DynamicReference
9253                        && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
9254                    {
9255                        Self::widen_target_preparation_to_sheet(
9256                            options.opaque_policy,
9257                            &mut scope,
9258                            &mut reasons,
9259                            reason,
9260                            &region.sheet,
9261                        )?;
9262                    } else {
9263                        Self::widen_target_preparation(
9264                            options.opaque_policy,
9265                            &mut scope,
9266                            &mut reasons,
9267                            reason,
9268                        )?;
9269                    }
9270                }
9271                let proven_sheet_local_dynamic =
9272                    Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9273                let placement = CellRef::new(
9274                    sheet_id,
9275                    Coord::from_excel(lease.row, lease.col, true, true),
9276                );
9277                let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
9278                    FormulaAstInput::Tree(ast),
9279                    placement,
9280                    Some(Arc::from(formula)),
9281                )?;
9282                self.target_preparation_checkpoint(options.deadline, 1)?;
9283                if ingested.dep_plan.dynamic {
9284                    if proven_sheet_local_dynamic {
9285                        Self::widen_target_preparation_to_sheet(
9286                            options.opaque_policy,
9287                            &mut scope,
9288                            &mut reasons,
9289                            OpaqueReason::DynamicReference,
9290                            &region.sheet,
9291                        )?;
9292                    } else {
9293                        Self::widen_target_preparation(
9294                            options.opaque_policy,
9295                            &mut scope,
9296                            &mut reasons,
9297                            OpaqueReason::DynamicReference,
9298                        )?;
9299                    }
9300                }
9301                for dep in &ingested.dep_plan.direct_cell_deps {
9302                    self.target_preparation_checkpoint(options.deadline, 1)?;
9303                    regions.push_back(PreparationRegion {
9304                        sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9305                        sheet_id: dep.sheet_id,
9306                        start_row: dep.coord.row() + 1,
9307                        start_col: dep.coord.col() + 1,
9308                        end_row: dep.coord.row() + 1,
9309                        end_col: dep.coord.col() + 1,
9310                    });
9311                }
9312                for range in &ingested.dep_plan.range_deps {
9313                    self.target_preparation_checkpoint(options.deadline, 1)?;
9314                    // `Current` is the sheet the staged formula lives on.
9315                    let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
9316                    else {
9317                        Self::widen_target_preparation(
9318                            options.opaque_policy,
9319                            &mut scope,
9320                            &mut reasons,
9321                            OpaqueReason::UnresolvedCrossSheetBinding,
9322                        )?;
9323                        continue;
9324                    };
9325                    regions.push_back(PreparationRegion {
9326                        sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9327                        sheet_id: dependency_sheet,
9328                        start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9329                        start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9330                        end_row: range
9331                            .end_row
9332                            .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9333                                bound.index + 1
9334                            }),
9335                        end_col: range
9336                            .end_col
9337                            .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9338                                bound.index + 1
9339                            }),
9340                    });
9341                }
9342                for name in ingested
9343                    .dep_plan
9344                    .resolved_named_refs
9345                    .iter()
9346                    .chain(&ingested.dep_plan.named_refs)
9347                {
9348                    self.target_preparation_checkpoint(options.deadline, 1)?;
9349                    if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
9350                        symbol_vertices.push_back(entry.vertex);
9351                    } else if self.graph.resolve_source_scalar_entry(name).is_none()
9352                        && self.graph.resolve_source_table_entry(name).is_none()
9353                    {
9354                        Self::widen_target_preparation(
9355                            options.opaque_policy,
9356                            &mut scope,
9357                            &mut reasons,
9358                            OpaqueReason::UnresolvedName,
9359                        )?;
9360                    }
9361                }
9362                for table in &ingested.dep_plan.table_refs {
9363                    self.target_preparation_checkpoint(options.deadline, 1)?;
9364                    if let Some(entry) = self.graph.resolve_table_entry(table) {
9365                        symbol_vertices.push_back(entry.vertex);
9366                    } else if self.graph.resolve_source_table_entry(table).is_none() {
9367                        Self::widen_target_preparation(
9368                            options.opaque_policy,
9369                            &mut scope,
9370                            &mut reasons,
9371                            OpaqueReason::UnresolvedTable,
9372                        )?;
9373                    }
9374                }
9375                selected_cells.push(
9376                    formualizer_common::RangeAddress::new(
9377                        region.sheet.clone(),
9378                        lease.row,
9379                        lease.col,
9380                        lease.row,
9381                        lease.col,
9382                    )
9383                    .expect("selected staged coordinates are valid"),
9384                );
9385                prepared.push(PreparedOrdinaryStagedFormula {
9386                    sheet: region.sheet.clone(),
9387                    sheet_id,
9388                    lease,
9389                    ast_id: Some(ingested.ast_id),
9390                    plan: Some(ingested.dep_plan),
9391                });
9392            }
9393
9394            if indexed_query_sheets.insert(region.sheet_id) {
9395                self.graph.prepare_sheet_index_for_query(region.sheet_id);
9396                let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
9397                    .saturating_mul(32);
9398                self.reserve_graph_source_scratch(bytes)?;
9399                discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
9400            }
9401            let spill_anchors = self.graph.spill_anchors_in_region(
9402                region.sheet_id,
9403                region.start_row - 1,
9404                region.start_col - 1,
9405                region.end_row - 1,
9406                region.end_col - 1,
9407            );
9408            for anchor in spill_anchors {
9409                self.target_preparation_checkpoint(options.deadline, 1)?;
9410                symbol_vertices.push_back(anchor);
9411            }
9412            let vertices = self.graph.vertices_in_region(
9413                region.sheet_id,
9414                region.start_row - 1,
9415                region.end_row - 1,
9416                region.start_col - 1,
9417                region.end_col - 1,
9418            );
9419            for vertex in vertices {
9420                self.target_preparation_checkpoint(options.deadline, 1)?;
9421                symbol_vertices.push_back(vertex);
9422            }
9423        }
9424
9425        #[cfg(test)]
9426        self.target_preparation_fault(
9427            crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
9428        )?;
9429
9430        if package_encountered {
9431            Self::widen_target_preparation(
9432                options.opaque_policy,
9433                &mut scope,
9434                &mut reasons,
9435                OpaqueReason::UnsupportedSourceSemantics,
9436            )?;
9437            self.target_preparation_checkpoint(options.deadline, 0)?;
9438            let selected_count = self.staged_formula_count();
9439            let selected_packages = self
9440                .staged_formulas
9441                .values()
9442                .filter_map(|staged| staged.deferred_package.as_ref())
9443                .map(|package| package.families.len() + package.partitioned_families.len())
9444                .sum();
9445            self.formula_parse_diagnostics.truncate(diagnostics_len);
9446            self.last_formula_ingest_report = report_len;
9447            #[cfg(test)]
9448            if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9449                hook();
9450            }
9451            self.target_preparation_checkpoint(options.deadline, 0)?;
9452            #[cfg(test)]
9453            self.target_preparation_fault(
9454                crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9455            )?;
9456            let current_revisions = self.preparation_revisions();
9457            if let Some(reason) = Self::preparation_revision_stale_reason(
9458                &assumptions,
9459                &current_revisions,
9460                &planning_requests,
9461                true,
9462            ) {
9463                return Err(Self::preparation_stale(
9464                    reason,
9465                    "target compatibility preparation plan is stale",
9466                ));
9467            }
9468            #[cfg(test)]
9469            self.target_preparation_fault(
9470                crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9471            )?;
9472            let commit_work_before = self
9473                .active_resource_ledger
9474                .as_ref()
9475                .map_or(0, |ledger| ledger.snapshot().work_charged);
9476            let commit_started = crate::instant::FzInstant::now();
9477            self.build_graph_all_unobserved()?;
9478            let commit_window = commit_started.elapsed();
9479            let ledger_after = self
9480                .active_resource_ledger
9481                .as_ref()
9482                .map(|ledger| ledger.snapshot());
9483            let actual_commit_work = ledger_after
9484                .map_or(0, |snapshot| snapshot.work_charged)
9485                .saturating_sub(commit_work_before);
9486            let observed_scratch_bytes = ledger_after
9487                .map_or(0, |snapshot| snapshot.scratch_peak)
9488                .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
9489            let revisions = assumptions.clone();
9490            let report = PreparedTargetGraphReport {
9491                request_id: request_id.unwrap_or_default(),
9492                requested_targets: targets.len(),
9493                normalized_regions: visited_regions.len(),
9494                normalized_target_list: normalized,
9495                selected_staged_cells: selected_count,
9496                selected_source_families: selected_packages,
9497                retained_staged_cells: self.staged_formula_count(),
9498                selected_cells,
9499                retained_cells: Vec::new(),
9500                widened_scope: PrepareScope::Workbook,
9501                widening_reasons: reasons,
9502                revisions,
9503                commit_window,
9504                estimated_scratch_bytes: discovery_scratch_reserved
9505                    .saturating_add((selected_count as u64).saturating_mul(256)),
9506                observed_scratch_bytes,
9507                estimated_commit_work: selected_count as u64,
9508                actual_commit_work,
9509                outcome: PreparationOutcome::CompatibilityPrepared,
9510            };
9511            self.observe_target_preparation_report(&report);
9512            return Ok(report);
9513        }
9514
9515        prepared.sort_by_key(|formula| formula.lease.insertion_order);
9516        for package in &prepared_packages {
9517            if let Some(points) = &package.selected_points {
9518                selected_cells.extend(points.iter().filter_map(|&(row, col)| {
9519                    formualizer_common::RangeAddress::new(&package.sheet, row, col, row, col).ok()
9520                }));
9521                continue;
9522            }
9523            selected_cells.extend(package.replay_records.iter().filter_map(|record| {
9524                formualizer_common::RangeAddress::new(
9525                    package.sheet.clone(),
9526                    record.row,
9527                    record.col,
9528                    record.row,
9529                    record.col,
9530                )
9531                .ok()
9532            }));
9533        }
9534        let (legacy_graph, planned_formula_count) =
9535            self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
9536        let new_vertices = legacy_graph.new_vertex_count();
9537        let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
9538            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9539        })?;
9540        let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
9541            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9542        })?;
9543        let current = self.graph.baseline_stats();
9544        let final_vertices = current
9545            .graph_vertex_count
9546            .checked_add(new_vertices)
9547            .ok_or_else(|| {
9548                crate::engine::ResourceLedgerError::Exhausted(
9549                    formualizer_common::ResourceExhaustionDetail {
9550                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9551                        limit: u64::MAX,
9552                        observed: u64::MAX,
9553                        request_id,
9554                    },
9555                )
9556                .into_excel_error()
9557            })?;
9558        let final_edges = current
9559            .graph_edge_count
9560            .checked_sub(removed_edges)
9561            .and_then(|count| count.checked_add(new_edges))
9562            .ok_or_else(|| {
9563                crate::engine::ResourceLedgerError::Exhausted(
9564                    formualizer_common::ResourceExhaustionDetail {
9565                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9566                        limit: u64::MAX,
9567                        observed: u64::MAX,
9568                        request_id,
9569                    },
9570                )
9571                .into_excel_error()
9572            })?;
9573        #[cfg(test)]
9574        self.target_preparation_fault(
9575            crate::engine::target_preparation::TargetPreparationFault::Admission,
9576        )?;
9577        let resource = |reason, limit: u64, observed: u64| {
9578            crate::engine::ResourceLedgerError::Exhausted(
9579                formualizer_common::ResourceExhaustionDetail {
9580                    reason,
9581                    limit,
9582                    observed,
9583                    request_id,
9584                },
9585            )
9586            .into_excel_error()
9587        };
9588        let admission = crate::engine::resource_ledger::GraphAdmission {
9589            final_vertices,
9590            final_edges,
9591            materialization_cells: planned_formula_count as u64,
9592            added_vertices: new_vertices,
9593            added_edges: new_edges,
9594        };
9595        let materialized_bytes = admission
9596            .materialized_graph_bytes()
9597            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
9598        if let Err(error) = self.preflight_graph_admission(admission) {
9599            if let formualizer_common::ExcelErrorExtra::Resource { detail } = error.extra() {
9600                self.observe_target_admission_failure(detail.reason);
9601            }
9602            return Err(error);
9603        }
9604        let selected_package_records = prepared_packages
9605            .iter()
9606            .map(|package| package.replay_records.len() as u64)
9607            .sum::<u64>();
9608        let planned_working_bytes = (prepared.len() as u64)
9609            .saturating_add(selected_package_records)
9610            .saturating_mul(256)
9611            .saturating_add((visited_regions.len() as u64).saturating_mul(128))
9612            .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
9613            .saturating_add((selected.len() as u64).saturating_mul(96))
9614            .saturating_add(materialized_bytes);
9615        let residual_scratch = selected_package_points
9616            .iter()
9617            .map(|(sheet, points)| {
9618                let source = self
9619                    .staged_formulas
9620                    .get(sheet)
9621                    .unwrap()
9622                    .deferred_package
9623                    .as_ref()
9624                    .unwrap();
9625                (source.families.len() as u64)
9626                    .saturating_mul(256)
9627                    .saturating_add(
9628                        source
9629                            .partitioned_families
9630                            .iter()
9631                            .map(|family| {
9632                                256u64
9633                                    .saturating_add(family.fragments.len() as u64 * 32)
9634                                    .saturating_add(family.legacy_members.len() as u64 * 32)
9635                            })
9636                            .sum::<u64>(),
9637                    )
9638                    // Direct complete domains need coordinate suppression, not
9639                    // a per-member residual split/proof or legacy AST reserve.
9640                    .saturating_add(points.len() as u64 * 32)
9641                    .saturating_add(
9642                        prepared_packages
9643                            .iter()
9644                            .filter(|package| &package.sheet == sheet)
9645                            .map(|package| package.replay_records.len() as u64 * 480)
9646                            .sum::<u64>(),
9647                    )
9648            })
9649            .sum::<u64>();
9650        let scratch_bytes = discovery_scratch_reserved
9651            .saturating_add(planned_working_bytes)
9652            .saturating_add(residual_scratch);
9653        let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
9654        if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
9655            self.observe_target_admission_failure(
9656                formualizer_common::ResourceExhaustionReason::ScratchMemory,
9657            );
9658            return Err(error);
9659        }
9660
9661        let mut residual_sources = BTreeMap::new();
9662        let mut residual_owners = BTreeMap::new();
9663        for (sheet, points) in &selected_package_points {
9664            let selected: BTreeMap<_, BTreeSet<_>> = prepared_packages
9665                .iter()
9666                .filter(|package| &package.sheet == sheet)
9667                .flat_map(|package| &package.replay_records)
9668                .filter_map(|record| {
9669                    record.partition_owner.or(record.family).map(|owner| {
9670                        (
9671                            owner,
9672                            crate::engine::SourceCoord {
9673                                row: record.row - 1,
9674                                col: record.col - 1,
9675                            },
9676                        )
9677                    })
9678                })
9679                .fold(BTreeMap::new(), |mut map, (owner, coord)| {
9680                    map.entry(owner).or_default().insert(coord);
9681                    map
9682                });
9683            let metadata_work = self
9684                .staged_formulas
9685                .get(sheet)
9686                .unwrap()
9687                .deferred_package
9688                .as_ref()
9689                .map_or(0, |source| {
9690                    source.families.len() as u64
9691                        + source
9692                            .partitioned_families
9693                            .iter()
9694                            .map(|family| {
9695                                1 + family.fragments.len() as u64 * family.fragments.len() as u64
9696                                    + family.legacy_members.len() as u64
9697                            })
9698                            .sum::<u64>()
9699                });
9700            let split_work = selected
9701                .values()
9702                .map(|points| points.len() as u64 * 128)
9703                .sum::<u64>();
9704            self.target_preparation_checkpoint(
9705                options.deadline,
9706                metadata_work
9707                    .saturating_add(split_work)
9708                    .saturating_add(points.len() as u64),
9709            )?;
9710            let source = self
9711                .staged_formulas
9712                .get(sheet)
9713                .unwrap()
9714                .deferred_package
9715                .as_ref()
9716                .unwrap();
9717            let complete: BTreeSet<_> = prepared_packages
9718                .iter()
9719                .filter(|package| &package.sheet == sheet)
9720                .flat_map(|package| package.complete_selections.iter().copied())
9721                .collect();
9722            let residual = source
9723                .residual_sources(&selected, &complete, &self.workbook_load_limits)
9724                .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
9725            residual_owners.insert(
9726                sheet.clone(),
9727                residual
9728                    .1
9729                    .iter()
9730                    .map(|family| family.source_id)
9731                    .collect::<BTreeSet<_>>(),
9732            );
9733            residual_sources.insert(sheet.clone(), residual);
9734        }
9735
9736        // Reserve residual suppression before the revision-validated commit window.
9737        // Hash-set insertion and point-index removal below cannot allocate.
9738        for (sheet, points) in &selected_package_points {
9739            let package = self
9740                .staged_formulas
9741                .get_mut(sheet)
9742                .unwrap()
9743                .deferred_package
9744                .as_mut()
9745                .unwrap();
9746            let owner_count = prepared_packages
9747                .iter()
9748                .filter(|p| &p.sheet == sheet)
9749                .flat_map(|p| &p.replay_records)
9750                .filter_map(|record| record.partition_owner.or(record.family))
9751                .filter(|owner| residual_owners[sheet].contains(owner))
9752                .count();
9753            package
9754                .consumed_members
9755                .try_reserve(owner_count)
9756                .map_err(|_| {
9757                    resource(
9758                        formualizer_common::ResourceExhaustionReason::ScratchMemory,
9759                        0,
9760                        owner_count as u64 * 24,
9761                    )
9762                })?;
9763            package.suppressed.try_reserve(points.len()).map_err(|_| {
9764                resource(
9765                    formualizer_common::ResourceExhaustionReason::ScratchMemory,
9766                    0,
9767                    points.len() as u64 * 16,
9768                )
9769            })?;
9770        }
9771
9772        let estimated_commit_duration = std::time::Duration::from_nanos(
9773            (new_vertices as u64)
9774                .saturating_add(new_edges as u64)
9775                .saturating_add(prepared.len() as u64)
9776                .saturating_add(selected_package_records)
9777                .max(1)
9778                .saturating_mul(100),
9779        );
9780        if options.deadline.is_some_and(|deadline| {
9781            std::time::Instant::now()
9782                .checked_add(estimated_commit_duration)
9783                .is_none_or(|finish| finish > deadline)
9784        }) {
9785            self.observe_target_admission_failure(
9786                formualizer_common::ResourceExhaustionReason::Deadline,
9787            );
9788            return Err(resource(
9789                formualizer_common::ResourceExhaustionReason::Deadline,
9790                0,
9791                1,
9792            ));
9793        }
9794        #[cfg(test)]
9795        if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9796            hook();
9797        }
9798        self.target_preparation_checkpoint(options.deadline, 0)?;
9799        #[cfg(test)]
9800        self.target_preparation_fault(
9801            crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9802        )?;
9803        let current_revisions = self.preparation_revisions();
9804        let staged_leases_match = prepared.iter().all(|formula| {
9805            self.staged_formula_index
9806                .lease_matches(&formula.sheet, formula.lease)
9807        }) && prepared_packages.iter().all(|package| {
9808            self.staged_formula_index
9809                .package_lease_matches(&package.sheet, package.lease)
9810        });
9811        let stale_reason = Self::preparation_revision_stale_reason(
9812            &assumptions,
9813            &current_revisions,
9814            &planning_requests,
9815            staged_leases_match,
9816        );
9817        if let Some(reason) = stale_reason {
9818            return Err(Self::preparation_stale(
9819                reason,
9820                "target graph preparation plan is stale",
9821            ));
9822        }
9823        #[cfg(test)]
9824        self.target_preparation_fault(
9825            crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9826        )?;
9827        self.graph
9828            .validate_prepared_legacy_graph_plan(&legacy_graph)
9829            .map_err(|error| {
9830                Self::preparation_stale(
9831                    formualizer_common::PreparationStaleReason::Graph,
9832                    format!("target graph preparation plan is stale: {error}"),
9833                )
9834            })?;
9835        #[cfg(test)]
9836        self.target_preparation_fault(
9837            crate::engine::target_preparation::TargetPreparationFault::Reservation,
9838        )?;
9839        self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
9840        self.formula_parse_diagnostics
9841            .try_reserve(pending_diagnostics.len())
9842            .map_err(|_| {
9843                resource(
9844                    formualizer_common::ResourceExhaustionReason::Admission,
9845                    pending_diagnostics.len() as u64,
9846                    pending_diagnostics.len() as u64,
9847                )
9848            })?;
9849        self.target_preparation_checkpoint(options.deadline, 0)?;
9850        #[cfg(test)]
9851        self.target_preparation_fault(
9852            crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
9853        )?;
9854
9855        let commit_started = crate::instant::FzInstant::now();
9856        let committed = self
9857            .graph
9858            .apply_prevalidated_legacy_graph_plan(legacy_graph);
9859        for formula in &prepared {
9860            let removed = self
9861                .staged_formulas
9862                .get_mut(&formula.sheet)
9863                .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
9864            debug_assert!(removed.is_some());
9865            let index_removed = self.staged_formula_index.remove(
9866                &formula.sheet,
9867                formula.lease.row,
9868                formula.lease.col,
9869            );
9870            debug_assert!(index_removed);
9871        }
9872        for package in &prepared_packages {
9873            if let Some(points) = &package.selected_points {
9874                let staged = self.staged_formulas.get_mut(&package.sheet).unwrap();
9875                let source = staged.deferred_package.as_mut().unwrap();
9876                source.consumed_engine = Some(Arc::clone(&self.source_formula_token));
9877                // The revision-validated graph commit establishes this exact source
9878                // ownership proof. Later edits retain the exclusion, not old text.
9879                source
9880                    .consumed_members
9881                    .extend(package.replay_records.iter().filter_map(|record| {
9882                        record
9883                            .partition_owner
9884                            .or(record.family)
9885                            .filter(|owner| residual_owners[&package.sheet].contains(owner))
9886                            .map(|owner| {
9887                                (
9888                                    owner,
9889                                    crate::engine::SourceCoord {
9890                                        row: record.row - 1,
9891                                        col: record.col - 1,
9892                                    },
9893                                )
9894                            })
9895                    }));
9896                source.suppressed.extend(points.iter().copied());
9897                source.source_accounted = true;
9898                self.staged_formula_index
9899                    .consume_package_points(&package.sheet, points);
9900                if source.suppressed.len() >= source.source_coordinates.len() {
9901                    staged.deferred_package = None;
9902                    self.staged_formula_index.set_package(&package.sheet, None);
9903                }
9904            } else {
9905                let removed = self
9906                    .staged_formulas
9907                    .get_mut(&package.sheet)
9908                    .and_then(|staged| staged.deferred_package.take());
9909                debug_assert!(removed.is_some());
9910                self.staged_formula_index.set_package(&package.sheet, None);
9911            }
9912        }
9913        for (sheet, (families, partitions)) in residual_sources {
9914            if let Some(source) = self
9915                .staged_formulas
9916                .get_mut(&sheet)
9917                .and_then(|staged| staged.deferred_package.as_mut())
9918            {
9919                source.families = families;
9920                source.partitioned_families = partitions;
9921                source
9922                    .consumed_members
9923                    .retain(|(owner, _)| residual_owners[&sheet].contains(owner));
9924                self.staged_formula_index.update_package_family_count(
9925                    &sheet,
9926                    source.families.len() + source.partitioned_families.len(),
9927                );
9928            }
9929        }
9930        let empty_sheets = self
9931            .staged_formulas
9932            .iter()
9933            .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
9934            .collect::<Vec<_>>();
9935        for sheet in empty_sheets {
9936            self.staged_formulas.remove(&sheet);
9937        }
9938        if committed > 0 {
9939            self.mark_topology_edited();
9940        }
9941        self.formula_parse_diagnostics.extend(pending_diagnostics);
9942        if !prepared.is_empty() || !prepared_packages.is_empty() {
9943            let mut ingest_delta = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
9944            ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
9945                prepared_packages
9946                    .iter()
9947                    .map(|package| {
9948                        (package.replay_records.len() as u64).saturating_add(
9949                            if package.selected_points.is_some() {
9950                                package.direct_cells
9951                            } else {
9952                                0
9953                            },
9954                        )
9955                    })
9956                    .sum::<u64>(),
9957            );
9958            ingest_delta.graph_formula_cells_materialized = committed as u64;
9959            ingest_delta.graph_vertices_created = new_vertices as u64;
9960            ingest_delta.graph_edges_created = new_edges as u64;
9961            for package in &prepared_packages {
9962                let source = &package.source_report;
9963                ingest_delta.source_formula_events = ingest_delta
9964                    .source_formula_events
9965                    .saturating_add(source.source_formula_events);
9966                ingest_delta.source_formula_records_spooled = ingest_delta
9967                    .source_formula_records_spooled
9968                    .saturating_add(source.source_formula_records_spooled);
9969                ingest_delta.source_spool_encoded_bytes = ingest_delta
9970                    .source_spool_encoded_bytes
9971                    .saturating_add(source.source_spool_encoded_bytes);
9972                ingest_delta.source_spool_peak_memory_bytes = ingest_delta
9973                    .source_spool_peak_memory_bytes
9974                    .max(source.source_spool_peak_memory_bytes);
9975                ingest_delta.source_spool_spilled_bytes = ingest_delta
9976                    .source_spool_spilled_bytes
9977                    .saturating_add(source.source_spool_spilled_bytes);
9978                ingest_delta.source_spool_spill_files = ingest_delta
9979                    .source_spool_spill_files
9980                    .saturating_add(source.source_spool_spill_files);
9981                ingest_delta.source_spool_replays = ingest_delta
9982                    .source_spool_replays
9983                    .saturating_add(source.source_spool_replays)
9984                    .saturating_add(package.spool_replays);
9985                ingest_delta.source_families_seen = ingest_delta
9986                    .source_families_seen
9987                    .saturating_add(source.families_seen);
9988                ingest_delta.source_family_cells_seen = ingest_delta
9989                    .source_family_cells_seen
9990                    .saturating_add(source.family_cells_seen);
9991                ingest_delta.source_family_shadow_eligible = ingest_delta
9992                    .source_family_shadow_eligible
9993                    .saturating_add(source.source_clean_families);
9994                ingest_delta.source_family_shadow_eligible_cells = ingest_delta
9995                    .source_family_shadow_eligible_cells
9996                    .saturating_add(source.source_clean_cells);
9997                ingest_delta.source_partitioned_families_seen = ingest_delta
9998                    .source_partitioned_families_seen
9999                    .saturating_add(source.source_fragmentable_families);
10000                ingest_delta.source_partition_holes = ingest_delta
10001                    .source_partition_holes
10002                    .saturating_add(source.source_hole_exclusions);
10003                ingest_delta.source_partition_ordinary_exceptions = ingest_delta
10004                    .source_partition_ordinary_exceptions
10005                    .saturating_add(source.source_ordinary_exclusions);
10006                ingest_delta.source_partition_surviving_cells = ingest_delta
10007                    .source_partition_surviving_cells
10008                    .saturating_add(source.source_fragmentable_cells);
10009                for (reason, count) in &source.fallback_reasons {
10010                    let total = ingest_delta
10011                        .fallback_reasons
10012                        .entry(reason.clone())
10013                        .or_default();
10014                    *total = total.saturating_add(*count);
10015                }
10016
10017                ingest_delta.source_family_fallback = ingest_delta
10018                    .source_family_fallback
10019                    .saturating_add(source.families_seen);
10020                ingest_delta.source_family_fallback_cells = ingest_delta
10021                    .source_family_fallback_cells
10022                    .saturating_add(source.family_cells_seen);
10023            }
10024            self.record_formula_ingest_report(ingest_delta);
10025        }
10026        let commit_window = commit_started.elapsed();
10027        let retained_cells = self
10028            .staged_formula_index
10029            .all_leases()
10030            .into_iter()
10031            .filter_map(|(sheet, lease)| {
10032                formualizer_common::RangeAddress::new(
10033                    sheet, lease.row, lease.col, lease.row, lease.col,
10034                )
10035                .ok()
10036            })
10037            .collect::<Vec<_>>();
10038        let observed_scratch_bytes = self
10039            .active_resource_ledger
10040            .as_ref()
10041            .map_or(0, |ledger| ledger.snapshot().scratch_peak)
10042            .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
10043        let committed_spans = 0u64;
10044        let selected_source_families = prepared_packages
10045            .iter()
10046            .filter(|package| package.selected_points.is_none())
10047            .map(|package| package.lease.family_count)
10048            .sum::<usize>()
10049            + prepared_packages
10050                .iter()
10051                .filter(|package| package.selected_points.is_some())
10052                .flat_map(|package| package.replay_records.iter())
10053                .filter_map(|record| record.partition_owner.or(record.family))
10054                .chain(
10055                    prepared_packages
10056                        .iter()
10057                        .flat_map(|package| package.complete_selections.iter().copied()),
10058                )
10059                .collect::<BTreeSet<_>>()
10060                .len();
10061        let selected_staged_cells = prepared.len().saturating_add(
10062            prepared_packages
10063                .iter()
10064                .map(|package| {
10065                    package
10066                        .selected_points
10067                        .as_ref()
10068                        .map_or(package.replay_records.len(), BTreeSet::len)
10069                })
10070                .sum::<usize>(),
10071        );
10072        let actual_commit_work = (new_vertices as u64)
10073            .saturating_add(new_edges as u64)
10074            .saturating_add(committed as u64)
10075            .saturating_add(committed_spans)
10076            .saturating_add(prepared.len() as u64)
10077            .saturating_add(
10078                prepared_packages
10079                    .iter()
10080                    .map(|package| {
10081                        package
10082                            .selected_points
10083                            .as_ref()
10084                            .map_or(1, |points| points.len() as u64)
10085                    })
10086                    .sum::<u64>(),
10087            );
10088        let report = PreparedTargetGraphReport {
10089            request_id: request_id.unwrap_or_default(),
10090            requested_targets: targets.len(),
10091            normalized_regions: visited_regions.len(),
10092            normalized_target_list: normalized,
10093            selected_staged_cells,
10094            selected_source_families,
10095            retained_staged_cells: self.staged_formula_count(),
10096            selected_cells,
10097            retained_cells,
10098            widened_scope: scope,
10099            widening_reasons: reasons,
10100            revisions: assumptions,
10101            commit_window,
10102            estimated_scratch_bytes: scratch_bytes,
10103            observed_scratch_bytes,
10104            estimated_commit_work: (new_vertices as u64)
10105                .saturating_add(new_edges as u64)
10106                .saturating_add(prepared.len() as u64)
10107                .saturating_add(selected_package_records),
10108            actual_commit_work,
10109            outcome: PreparationOutcome::Prepared,
10110        };
10111        self.observe_target_preparation_report(&report);
10112        Ok(report)
10113    }
10114
10115    /// Build graph for all staged formulas.
10116    pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
10117        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
10118            engine.build_graph_all_unobserved()
10119        })
10120    }
10121
10122    fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
10123        let selected = self.staged_formula_count();
10124        let started = crate::instant::FzInstant::now();
10125        self.resource_checkpoint(selected as u64)?;
10126        let scratch_bytes = (selected as u64).saturating_mul(256);
10127        let result = self.with_request_scratch(scratch_bytes, |engine| {
10128            let index_snapshot = engine.staged_formula_index.clone();
10129            let collected = std::mem::take(&mut engine.staged_formulas)
10130                .into_iter()
10131                .collect();
10132            engine.staged_formula_index.clear_all();
10133            engine.build_graph_from_staged_batches(collected, false, index_snapshot)
10134        });
10135        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
10136        result
10137    }
10138
10139    /// Build graph for specific sheets (consuming only those staged entries).
10140    pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
10141        &mut self,
10142        sheets: I,
10143    ) -> Result<(), formualizer_parse::ExcelError> {
10144        let mut sheets = sheets.into_iter();
10145        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
10146            let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
10147            engine.with_request_scratch(name_scratch, |engine| {
10148                // Allocation failure follows the baseline process-fatal policy; it is not a
10149                // recoverable resource error or a new staged-preparation route.
10150                let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
10151                engine.charge_bounded_work(names.len() as u64)?;
10152                engine.build_graph_for_sheet_names_unobserved(names)
10153            })
10154        })
10155    }
10156
10157    fn build_graph_for_sheet_names_unobserved(
10158        &mut self,
10159        sheets: Vec<String>,
10160    ) -> Result<(), formualizer_parse::ExcelError> {
10161        let started = crate::instant::FzInstant::now();
10162        let selected = sheets
10163            .iter()
10164            .filter_map(|sheet| self.staged_formulas.get(sheet))
10165            .map(StagedSheet::len)
10166            .sum::<usize>();
10167        self.resource_checkpoint(selected as u64)?;
10168        let scratch_bytes = (selected as u64).saturating_mul(256);
10169        self.reserve_request_scratch(scratch_bytes)?;
10170        let index_snapshot = self.staged_formula_index.clone();
10171        let mut collected = Vec::new();
10172        for sheet in sheets {
10173            if let Some(staged) = self.staged_formulas.remove(&sheet) {
10174                self.index_removed_staged_sheet(&sheet, &staged);
10175                collected.push((sheet, staged));
10176            }
10177        }
10178        let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
10179        self.release_request_scratch(scratch_bytes);
10180        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
10181        result
10182    }
10183
10184    fn build_graph_from_staged_batches(
10185        &mut self,
10186        collected: StagedFormulaBatches,
10187        share_parse_cache_across_sheets: bool,
10188        staged_index_snapshot: StagedFormulaIndex,
10189    ) -> Result<(), formualizer_parse::ExcelError> {
10190        if collected.is_empty() {
10191            return Ok(());
10192        }
10193        for (sheet, _) in &collected {
10194            let _ = self.add_sheet(sheet);
10195        }
10196
10197        let diagnostics_len = self.formula_parse_diagnostics.len();
10198        let mut collected = collected;
10199        let prepared = match self
10200            .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
10201        {
10202            Ok(prepared) => prepared,
10203            Err(error) => {
10204                self.formula_parse_diagnostics.truncate(diagnostics_len);
10205                for (sheet, staged) in collected {
10206                    self.restore_staged_sheet(sheet, staged);
10207                }
10208                self.staged_formula_index = staged_index_snapshot;
10209                return Err(error);
10210            }
10211        };
10212        let (ordinary, compressed, direct, may_fail) = prepared;
10213
10214        // A first build (no formula in the graph yet) goes through the eager
10215        // first load's machinery: the builder pre-allocates each column's
10216        // targets as one id run, installs family members as virtual runs and
10217        // builds the authority once at the end (decision 26).
10218        // The first load's builder plans and applies one chunk at a time, so
10219        // a planning error would leave earlier chunks in the graph; the
10220        // incremental path plans everything first. Check every distinct
10221        // formula first, in the incremental path's order (it reports the
10222        // same first error and leaves the graph untouched).
10223        let first_build = if self.deferred_build_can_be_first_load() {
10224            if let Err(error) = self.check_staged_formula_plans(
10225                ordinary
10226                    .iter()
10227                    .chain(compressed.iter().map(|(batch, _)| batch)),
10228                &may_fail,
10229            ) {
10230                self.formula_parse_diagnostics.truncate(diagnostics_len);
10231                for (sheet, staged) in collected {
10232                    self.restore_staged_sheet(sheet, staged);
10233                }
10234                self.staged_formula_index = staged_index_snapshot;
10235                return Err(error);
10236            }
10237            self.deferred_build_as_first_load()
10238        } else {
10239            None
10240        };
10241        let built_sheets: Vec<String> = if first_build.is_some() {
10242            collected.iter().map(|(sheet, _)| sheet.clone()).collect()
10243        } else {
10244            Vec::new()
10245        };
10246
10247        // Keep the original source/spool alive through every fallible ingestion route.
10248        // Graph admission may have committed a prefix; replay replaces those placements
10249        // rather than treating their cached values as authoritative source.
10250        let result = (|| {
10251            if !ordinary.is_empty() {
10252                self.ingest_formula_batches(ordinary)?;
10253            }
10254            if !compressed.is_empty() {
10255                self.ingest_compressed_formula_source_batches(compressed)?;
10256            }
10257            if !direct.is_empty() {
10258                self.finish_compressed_formula_sources(direct)?;
10259            }
10260            Ok(())
10261        })();
10262        if let Some(saved) = first_build {
10263            self.leave_deferred_first_load(saved, &built_sheets);
10264        }
10265        if let Err(error) = result {
10266            self.formula_parse_diagnostics.truncate(diagnostics_len);
10267            for (sheet, staged) in collected {
10268                self.restore_staged_sheet(sheet, staged);
10269            }
10270            self.staged_formula_index = staged_index_snapshot;
10271            return Err(error);
10272        }
10273        self.dedup_formula_parse_diagnostics_since(diagnostics_len);
10274        Ok(())
10275    }
10276
10277    /// Enter the first-load ingest mode for a deferred build when the graph
10278    /// holds no formula yet and no load is in progress: the settings the
10279    /// calamine loader uses for its eager first load (lazy sheet index, no
10280    /// small-range expansion, first-load fast path). Returns the settings
10281    /// to restore, or `None` when the build takes the incremental path.
10282    fn deferred_build_can_be_first_load(&self) -> bool {
10283        !self.graph.first_load_assume_new()
10284            && !self.graph_admission_enabled()
10285            && self.graph.formula_vertex_count() == 0
10286    }
10287
10288    /// Plan each distinct formula of `batches` that may fail planning once
10289    /// (a member is planned through its template, staged before it) and
10290    /// drop the plans: the first planning error, in the order the
10291    /// incremental ingest meets it.
10292    fn check_staged_formula_plans<'b>(
10293        &mut self,
10294        batches: impl Iterator<Item = &'b FormulaIngestBatch>,
10295        may_fail: &FxHashSet<crate::engine::arena::AstNodeId>,
10296    ) -> Result<(), ExcelError> {
10297        if may_fail.is_empty() {
10298            return Ok(());
10299        }
10300        let mut seen: FxHashSet<(SheetId, crate::engine::arena::AstNodeId)> = FxHashSet::default();
10301        for batch in batches {
10302            let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
10303                ExcelError::new(ExcelErrorKind::Ref)
10304                    .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
10305            })?;
10306            let mut pipeline = self.ingest_pipeline();
10307            for record in &batch.formulas {
10308                if record.member_anchor.is_some()
10309                    || !may_fail.contains(&record.ast_id)
10310                    || !seen.insert((sheet_id, record.ast_id))
10311                {
10312                    continue;
10313                }
10314                let placement = CellRef::new(
10315                    sheet_id,
10316                    Coord::from_excel(record.row, record.col, true, true),
10317                );
10318                pipeline.ingest_formula(
10319                    FormulaAstInput::RawArena(record.ast_id),
10320                    placement,
10321                    None,
10322                )?;
10323            }
10324        }
10325        Ok(())
10326    }
10327
10328    fn deferred_build_as_first_load(&mut self) -> Option<(crate::engine::SheetIndexMode, usize)> {
10329        if !self.deferred_build_can_be_first_load() {
10330            return None;
10331        }
10332        let saved = (
10333            self.graph.get_config().sheet_index_mode,
10334            self.config.range_expansion_limit,
10335        );
10336        self.graph
10337            .set_sheet_index_mode(crate::engine::SheetIndexMode::Lazy);
10338        self.config.range_expansion_limit = 0;
10339        self.graph.set_first_load_assume_new(true);
10340        self.graph.reset_ensure_touched();
10341        Some(saved)
10342    }
10343
10344    /// Leave the first-load mode of [`Self::deferred_build_as_first_load`]:
10345    /// the authority is built once here, as at the end of an eager load.
10346    fn leave_deferred_first_load(
10347        &mut self,
10348        (index_mode, range_limit): (crate::engine::SheetIndexMode, usize),
10349        sheets: &[String],
10350    ) {
10351        self.graph.set_first_load_assume_new(false);
10352        self.graph.reset_ensure_touched();
10353        self.graph.set_sheet_index_mode(index_mode);
10354        self.config.range_expansion_limit = range_limit;
10355        for sheet in sheets {
10356            self.graph.finalize_sheet_index(sheet);
10357        }
10358    }
10359
10360    fn prepare_staged_formula_batches(
10361        &mut self,
10362        collected: &mut StagedFormulaBatches,
10363        share_parse_cache_across_sheets: bool,
10364    ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
10365        let mut ordinary = Vec::new();
10366        let mut compressed = Vec::new();
10367        let mut direct = Vec::new();
10368        let mut may_fail = FxHashSet::default();
10369        let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
10370            rustc_hash::FxHashMap::default();
10371        cache.reserve(4096);
10372
10373        for (sheet, staged) in collected {
10374            if !share_parse_cache_across_sheets {
10375                cache.clear();
10376            }
10377            // Load-time family grouping, as the eager first load does:
10378            // relative copies of the formula above (or to the left) become
10379            // members of its family and are never interned.
10380            let mut grouper = crate::engine::FormulaFamilyGrouper::new();
10381            // The staged texts, then the deferred package's replayed ones,
10382            // are walked once (no combined copy: a replay can be the whole
10383            // sheet).
10384            let mut replayed_records = Vec::new();
10385            let deferred_source = None;
10386            let mut deferred_fallback = None;
10387            if let Some(package) = staged.deferred_package.as_mut() {
10388                if package.sheet_name != *sheet {
10389                    return Err(ExcelError::new(ExcelErrorKind::Value)
10390                        .with_message("deferred formula package sheet mismatch"));
10391                }
10392                let eligible: Vec<_> = package
10393                    .families
10394                    .iter()
10395                    .filter(|family| !package.invalidated.contains(&family.source_id))
10396                    .cloned()
10397                    .collect();
10398                let eligible_partitions: Vec<_> = package
10399                    .partitioned_families
10400                    .iter()
10401                    .filter(|family| !package.invalidated.contains(&family.source_id))
10402                    .cloned()
10403                    .collect();
10404
10405                let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
10406                for partition in &eligible_partitions {
10407                    replay_disposition
10408                        .register_partition(partition, false)
10409                        .map_err(|reason| {
10410                            ExcelError::new(ExcelErrorKind::Value).with_message(reason)
10411                        })?;
10412                }
10413                replay_disposition
10414                    .extend_suppressed_excel_coords(package.suppressed.iter().copied());
10415                let replayed = package
10416                    .replay
10417                    .lock()
10418                    .map_err(|_| {
10419                        ExcelError::new(ExcelErrorKind::Value)
10420                            .with_message("deferred formula spool lock poisoned")
10421                    })?
10422                    .replay(&replay_disposition)
10423                    .map_err(|message| {
10424                        ExcelError::new(ExcelErrorKind::Value).with_message(message)
10425                    })?;
10426                replayed_records = replayed;
10427                let mut report = package.accounting_report();
10428                report.source_spool_replays = report.source_spool_replays.saturating_add(1);
10429                deferred_fallback = Some((report, package.families.clone(), eligible_partitions));
10430            }
10431
10432            let n_entries = staged.entries.len() + replayed_records.len();
10433            let entries = staged
10434                .entries
10435                .iter()
10436                .cloned()
10437                .map(|(row, col, text)| (row, col, text, None))
10438                .chain(replayed_records.into_iter().map(|record| {
10439                    (
10440                        record.row,
10441                        record.col,
10442                        record.text,
10443                        Some((record.source_order, record.family, record.partition_owner)),
10444                    )
10445                }));
10446            let mut formulas = Vec::with_capacity(n_entries);
10447            let staged_order_base = u64::MAX.saturating_sub(n_entries as u64);
10448            for (entry_index, (row, col, txt, source_proof)) in entries.enumerate() {
10449                let key = if txt.starts_with('=') {
10450                    txt
10451                } else {
10452                    format!("={txt}")
10453                };
10454                grouper.note_formula();
10455                // A shared-formula family's records name their family.
10456                let shared_key = source_proof
10457                    .and_then(|(_, family, _)| family)
10458                    .map(|family| u64::from(family.source_index));
10459                let staged_record = if let Some(cached) = cache.get(&key) {
10460                    grouper.note_parse_cache_hit();
10461                    cached.map(|ast_id| {
10462                        self.note_staged_formula(&mut grouper, row, col, ast_id);
10463                        FormulaIngestRecord::new(row, col, ast_id, Some(Arc::<str>::from(key)))
10464                    })
10465                } else if let Some(record) =
10466                    self.stage_relocated_text(&mut grouper, row, col, &key, shared_key)
10467                {
10468                    Some(record)
10469                } else {
10470                    grouper.note_parse(key.len());
10471                    let parsed = formualizer_parse::parser::parse(&key);
10472                    let parsed_text = parsed.is_ok();
10473                    let parsed = match parsed {
10474                        Ok(parsed) => Some(parsed),
10475                        Err(error) => self.handle_formula_parse_error(
10476                            sheet,
10477                            row,
10478                            col,
10479                            &key,
10480                            error.to_string(),
10481                        )?,
10482                    };
10483                    match parsed {
10484                        Some(ast) => {
10485                            let record = self.stage_formula_ast(&mut grouper, row, col, &ast, None);
10486                            if parsed_text {
10487                                self.note_parsed_text(
10488                                    &mut grouper,
10489                                    row,
10490                                    col,
10491                                    &key,
10492                                    &ast,
10493                                    &record,
10494                                    shared_key,
10495                                );
10496                            }
10497                            if !record.is_family_member() && self.formula_may_fail_planning(&ast) {
10498                                may_fail.insert(record.ast_id);
10499                            }
10500                            // A member's text is not worth caching: relative
10501                            // copies do not repeat their text.
10502                            if record.is_family_member() {
10503                                Some(record)
10504                            } else {
10505                                let ast_id = record.ast_id;
10506                                cache.insert(key.clone(), Some(ast_id));
10507                                Some(FormulaIngestRecord::new(
10508                                    row,
10509                                    col,
10510                                    ast_id,
10511                                    Some(Arc::<str>::from(key)),
10512                                ))
10513                            }
10514                        }
10515                        None => {
10516                            cache.insert(key, None);
10517                            None
10518                        }
10519                    }
10520                };
10521
10522                if let Some(mut formula) = staged_record {
10523                    if let Some((order, family, owner)) = source_proof {
10524                        formula = formula.with_source_proof(order, family, owner);
10525                    } else if deferred_source.is_some() {
10526                        formula = formula.with_source_proof(
10527                            crate::engine::SourceFormulaOrder::new(
10528                                staged_order_base.saturating_add(entry_index as u64),
10529                            ),
10530                            None,
10531                            None,
10532                        );
10533                    }
10534                    formulas.push(formula);
10535                }
10536            }
10537
10538            self.finish_family_grouper(&mut grouper);
10539            let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
10540            if let Some((report, preparation)) = deferred_source {
10541                direct.push((batch, report, preparation));
10542            } else if let Some((report, families, partitions)) = deferred_fallback {
10543                let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
10544                    batch.sheet_name.clone(),
10545                    report,
10546                    families,
10547                    partitions,
10548                );
10549                compressed.push((batch, source_batch));
10550            } else if !batch.is_empty() {
10551                ordinary.push(batch);
10552            }
10553        }
10554        if std::env::var("FZ_DEBUG_LOAD")
10555            .ok()
10556            .is_some_and(|v| v != "0")
10557        {
10558            eprintln!(
10559                "[fz][families] deferred build: {}",
10560                self.source_family_counters.debug_line()
10561            );
10562        }
10563        Ok((ordinary, compressed, direct, may_fail))
10564    }
10565
10566    /// Whether planning `ast` can fail: a reference to a sheet that does not
10567    /// exist, an external, 3-D or table reference, or a reversed range.
10568    /// Unqualified cell and range references, names, literals and calls
10569    /// always plan.
10570    fn formula_may_fail_planning(&self, ast: &formualizer_parse::parser::ASTNode) -> bool {
10571        use formualizer_parse::parser::{ASTNodeType, ReferenceType};
10572        let sheet_missing = |sheet: &Option<String>| {
10573            sheet
10574                .as_deref()
10575                .is_some_and(|s| self.graph.sheet_id(s).is_none())
10576        };
10577        match &ast.node_type {
10578            ASTNodeType::Literal(_) | ASTNodeType::Omitted => false,
10579            ASTNodeType::Reference { reference, .. } => match reference {
10580                ReferenceType::Cell { sheet, .. } => sheet_missing(sheet),
10581                ReferenceType::Range {
10582                    sheet,
10583                    start_row,
10584                    start_col,
10585                    end_row,
10586                    end_col,
10587                    ..
10588                } => {
10589                    sheet_missing(sheet)
10590                        || matches!((start_row, end_row), (Some(a), Some(b)) if a > b)
10591                        || matches!((start_col, end_col), (Some(a), Some(b)) if a > b)
10592                }
10593                ReferenceType::NamedRange(_) => false,
10594                _ => true,
10595            },
10596            ASTNodeType::UnaryOp { expr, .. } => self.formula_may_fail_planning(expr),
10597            ASTNodeType::BinaryOp { left, right, .. } => {
10598                self.formula_may_fail_planning(left) || self.formula_may_fail_planning(right)
10599            }
10600            ASTNodeType::Function { args, .. } => {
10601                args.iter().any(|a| self.formula_may_fail_planning(a))
10602            }
10603            ASTNodeType::Call { callee, args } => {
10604                self.formula_may_fail_planning(callee)
10605                    || args.iter().any(|a| self.formula_may_fail_planning(a))
10606            }
10607            ASTNodeType::Array(rows) => rows
10608                .iter()
10609                .flatten()
10610                .any(|a| self.formula_may_fail_planning(a)),
10611        }
10612    }
10613
10614    /// Begin bulk Arrow ingest for base values (Phase A)
10615    pub fn begin_bulk_ingest_arrow(
10616        &mut self,
10617    ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
10618        crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
10619    }
10620
10621    /// Begin bulk updates to Arrow store (Phase C)
10622    pub fn begin_bulk_update_arrow(
10623        &mut self,
10624    ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
10625        crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
10626    }
10627
10628    fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
10629        self.graph.sheet_id(sheet).ok_or(
10630            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
10631                name: sheet.to_string(),
10632                reason: "Unknown sheet".to_string(),
10633            },
10634        )
10635    }
10636
10637    fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
10638        if row_1based == 0 {
10639            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10640        }
10641        Ok(row_1based - 1)
10642    }
10643
10644    fn normalize_row_range_1based(
10645        start_row_1based: u32,
10646        end_row_1based: u32,
10647    ) -> Result<(u32, u32), crate::engine::EditorError> {
10648        if start_row_1based == 0 || end_row_1based == 0 {
10649            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10650        }
10651        if start_row_1based > end_row_1based {
10652            return Err(crate::engine::EditorError::TransactionFailed {
10653                reason: "Row range start is greater than end".to_string(),
10654            });
10655        }
10656        Ok((start_row_1based - 1, end_row_1based - 1))
10657    }
10658
10659    fn invalidate_row_visibility_mask_cache(&self) {
10660        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
10661            cache.clear();
10662        }
10663    }
10664
10665    /// Hidden rows changed: drop the cached masks and dirty the formulas
10666    /// that read row visibility (SUBTOTAL/AGGREGATE).
10667    fn row_visibility_changed(&mut self) {
10668        self.invalidate_row_visibility_mask_cache();
10669        self.graph.mark_row_visibility_readers_dirty();
10670    }
10671
10672    fn set_row_hidden_by_sheet_id(
10673        &mut self,
10674        sheet_id: SheetId,
10675        row0: u32,
10676        hidden: bool,
10677        source: RowVisibilitySource,
10678    ) -> bool {
10679        let changed = {
10680            let state = self.row_visibility.entry(sheet_id).or_default();
10681            state.set_row_hidden(row0, hidden, source)
10682        };
10683
10684        let remove_entry = self
10685            .row_visibility
10686            .get(&sheet_id)
10687            .map(|state| state.is_empty())
10688            .unwrap_or(false);
10689        if remove_entry {
10690            self.row_visibility.remove(&sheet_id);
10691        }
10692
10693        if changed {
10694            self.row_visibility_changed();
10695        }
10696
10697        changed
10698    }
10699
10700    fn set_rows_hidden_by_sheet_id(
10701        &mut self,
10702        sheet_id: SheetId,
10703        start_row0: u32,
10704        end_row0: u32,
10705        hidden: bool,
10706        source: RowVisibilitySource,
10707    ) -> bool {
10708        let changed = {
10709            let state = self.row_visibility.entry(sheet_id).or_default();
10710            state.set_rows_hidden(start_row0, end_row0, hidden, source)
10711        };
10712
10713        let remove_entry = self
10714            .row_visibility
10715            .get(&sheet_id)
10716            .map(|state| state.is_empty())
10717            .unwrap_or(false);
10718        if remove_entry {
10719            self.row_visibility.remove(&sheet_id);
10720        }
10721
10722        if changed {
10723            self.row_visibility_changed();
10724        }
10725
10726        changed
10727    }
10728
10729    fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
10730        if count == 0 {
10731            return;
10732        }
10733        let mut changed = false;
10734        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10735            changed = state.insert_rows(before0, count);
10736            state.is_empty()
10737        } else {
10738            false
10739        };
10740        if remove_entry {
10741            self.row_visibility.remove(&sheet_id);
10742        }
10743        if changed {
10744            self.row_visibility_changed();
10745        }
10746    }
10747
10748    fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
10749        if count == 0 {
10750            return;
10751        }
10752        let mut changed = false;
10753        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10754            changed = state.delete_rows(start0, count);
10755            state.is_empty()
10756        } else {
10757            false
10758        };
10759        if remove_entry {
10760            self.row_visibility.remove(&sheet_id);
10761        }
10762        if changed {
10763            self.row_visibility_changed();
10764        }
10765    }
10766
10767    fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10768        if let crate::engine::ChangeEvent::SetRowVisibility {
10769            sheet_id,
10770            row0,
10771            source,
10772            old_hidden,
10773            ..
10774        } = event
10775        {
10776            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
10777        }
10778    }
10779
10780    fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10781        if let crate::engine::ChangeEvent::SetRowVisibility {
10782            sheet_id,
10783            row0,
10784            source,
10785            new_hidden,
10786            ..
10787        } = event
10788        {
10789            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
10790        }
10791    }
10792
10793    fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10794        for event in events.iter().rev() {
10795            self.apply_inverse_row_visibility_event(event);
10796        }
10797    }
10798
10799    fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10800        for event in events {
10801            self.apply_forward_row_visibility_event(event);
10802        }
10803    }
10804
10805    fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10806        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10807            sheet,
10808            row,
10809            col,
10810            old,
10811            ..
10812        } = event
10813        {
10814            self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
10815        }
10816    }
10817
10818    fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10819        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10820            sheet,
10821            row,
10822            col,
10823            new,
10824            ..
10825        } = event
10826        {
10827            self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
10828        }
10829    }
10830
10831    /// Set a single cell's staged formula text to `target` (clearing it when
10832    /// `None`). Used by undo/redo replay of per-cell staged-formula deltas.
10833    fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
10834        match target {
10835            Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
10836            None => {
10837                self.clear_staged_formula_text(sheet, row, col);
10838            }
10839        }
10840    }
10841
10842    pub fn set_row_hidden(
10843        &mut self,
10844        sheet: &str,
10845        row_1based: u32,
10846        hidden: bool,
10847        source: RowVisibilitySource,
10848    ) -> Result<(), crate::engine::EditorError> {
10849        self.observe_function_semantic_epoch()
10850            .map_err(crate::engine::EditorError::Excel)?;
10851        self.observe_function_semantic_epoch()
10852            .map_err(crate::engine::EditorError::Excel)?;
10853        let sheet_id = self.ensure_known_sheet_id(sheet)?;
10854        let row0 = Self::normalize_row_1based(row_1based)?;
10855        if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
10856            self.record_structural_change(StructuralScope::Region(Region::whole_row(
10857                sheet_id, row0,
10858            )));
10859            self.mark_data_edited();
10860        }
10861        Ok(())
10862    }
10863
10864    pub fn set_rows_hidden(
10865        &mut self,
10866        sheet: &str,
10867        start_row_1based: u32,
10868        end_row_1based: u32,
10869        hidden: bool,
10870        source: RowVisibilitySource,
10871    ) -> Result<(), crate::engine::EditorError> {
10872        let sheet_id = self.ensure_known_sheet_id(sheet)?;
10873        let (start_row0, end_row0) =
10874            Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
10875        if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
10876            if start_row0 == end_row0 {
10877                self.record_structural_change(StructuralScope::Region(Region::whole_row(
10878                    sheet_id, start_row0,
10879                )));
10880            } else {
10881                self.record_structural_change(StructuralScope::Sheet(sheet_id));
10882            }
10883            self.mark_data_edited();
10884        }
10885        Ok(())
10886    }
10887
10888    pub fn is_row_hidden(
10889        &self,
10890        sheet: &str,
10891        row_1based: u32,
10892        source: Option<RowVisibilitySource>,
10893    ) -> Option<bool> {
10894        let sheet_id = self.graph.sheet_id(sheet)?;
10895        let row0 = row_1based.checked_sub(1)?;
10896        Some(
10897            self.row_visibility
10898                .get(&sheet_id)
10899                .map(|state| state.is_row_hidden(row0, source))
10900                .unwrap_or(false),
10901        )
10902    }
10903
10904    pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
10905        let sheet_id = self.graph.sheet_id(sheet)?;
10906        Some(
10907            self.row_visibility
10908                .get(&sheet_id)
10909                .map(|state| state.version())
10910                .unwrap_or(0),
10911        )
10912    }
10913
10914    /// Cells of a sheet-backed `view` whose formula (own or a family
10915    /// template) calls SUBTOTAL, or AGGREGATE when `include_aggregate`, as
10916    /// sorted, disjoint `(col, first_row, last_row)` offset intervals within
10917    /// the view. Each formula's call bits are precomputed in the AST arena;
10918    /// a filled-down family is checked once per run, not once per cell.
10919    fn nested_subtotal_cells_for_view(
10920        &self,
10921        view: &RangeView<'_>,
10922        include_aggregate: bool,
10923    ) -> Option<Vec<(usize, usize, usize)>> {
10924        if !view.is_sheet_backed() {
10925            return None;
10926        }
10927        let (rows, cols) = view.dims();
10928        if rows == 0 || cols == 0 {
10929            return Some(Vec::new());
10930        }
10931        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10932        let (r0, c0) = (view.start_row(), view.start_col());
10933        let mask = if include_aggregate {
10934            crate::engine::arena::SUBTOTAL_CALL | crate::engine::arena::AGGREGATE_CALL
10935        } else {
10936            crate::engine::arena::SUBTOTAL_CALL
10937        };
10938        let ds = self.graph.data_store();
10939        let mut found: Vec<(usize, usize, usize)> = self
10940            .graph
10941            .formula_intervals_in_region(
10942                sheet_id,
10943                r0 as u32,
10944                (r0 + rows - 1) as u32,
10945                c0 as u32,
10946                (c0 + cols - 1) as u32,
10947                |template| ds.ast_subtotal_calls(template) & mask != 0,
10948            )
10949            .into_iter()
10950            .map(|(col, lo, hi)| (col as usize - c0, lo as usize - r0, hi as usize - r0))
10951            .collect();
10952        found.sort_unstable();
10953        // Merge touching or overlapping intervals of a column so a binary
10954        // search finds the one covering a cell.
10955        let mut out: Vec<(usize, usize, usize)> = Vec::with_capacity(found.len());
10956        for (col, lo, hi) in found {
10957            match out.last_mut() {
10958                Some(last) if last.0 == col && lo <= last.2 + 1 => last.2 = last.2.max(hi),
10959                _ => out.push((col, lo, hi)),
10960            }
10961        }
10962        Some(out)
10963    }
10964
10965    fn build_row_visibility_mask_for_view(
10966        &self,
10967        view: &RangeView<'_>,
10968        mode: VisibilityMaskMode,
10969    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
10970        let sheet_rows = view.sheet().nrows as usize;
10971        if sheet_rows == 0 || view.start_row() >= sheet_rows {
10972            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
10973        }
10974
10975        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10976        let start_row0 = view.start_row() as u32;
10977        let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
10978        let version = self
10979            .row_visibility
10980            .get(&sheet_id)
10981            .map(|state| state.version())
10982            .unwrap_or(0);
10983        let key = VisibilityMaskCacheKey {
10984            sheet_id,
10985            start_row0,
10986            end_row0,
10987            mode,
10988            version,
10989        };
10990
10991        if let Ok(cache) = self.row_visibility_mask_cache.read()
10992            && let Some(mask) = cache.get(&key)
10993        {
10994            #[cfg(test)]
10995            visibility_mask_test_hooks::inc_hit();
10996            return Some(mask.clone());
10997        }
10998
10999        #[cfg(test)]
11000        visibility_mask_test_hooks::inc_miss();
11001
11002        let state = self.row_visibility.get(&sheet_id);
11003        let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
11004        for row0 in start_row0..=end_row0 {
11005            let manual_hidden = state
11006                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
11007                .unwrap_or(false);
11008            let filter_hidden = state
11009                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
11010                .unwrap_or(false);
11011
11012            let include = match mode {
11013                VisibilityMaskMode::IncludeAll => true,
11014                VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
11015                VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
11016                VisibilityMaskMode::ExcludeManualOrFilterHidden => {
11017                    !(manual_hidden || filter_hidden)
11018                }
11019            };
11020            out.push(include);
11021        }
11022
11023        let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
11024        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
11025            const MAX_CACHE_ENTRIES: usize = 4096;
11026            if cache.len() >= MAX_CACHE_ENTRIES {
11027                cache.clear();
11028                #[cfg(test)]
11029                visibility_mask_test_hooks::inc_eviction();
11030            }
11031            cache.insert(key, mask.clone());
11032        }
11033
11034        Some(mask)
11035    }
11036
11037    fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
11038        let changes =
11039            crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
11040        let global_changed = changes.epoch != self.function_semantic_epoch_seen;
11041        let provider_revision = self.resolver.planning_semantic_revision();
11042        let provider_changed = provider_revision != self.function_provider_revision_seen;
11043        if !global_changed && !provider_changed {
11044            return Ok(false);
11045        }
11046
11047        let changed = !changes.keys.is_empty();
11048        if global_changed && changed || provider_changed {
11049            self.cached_static_schedule = None;
11050            self.recent_schedules.clear();
11051            self.base_schedule = None;
11052        }
11053        self.function_semantic_epoch_seen = changes.epoch;
11054        self.function_provider_revision_seen = provider_revision;
11055        Ok(false)
11056    }
11057
11058    pub(crate) fn ast_uses_changed_function(
11059        ast: &ASTNode,
11060        changed: &BTreeSet<(String, String)>,
11061    ) -> bool {
11062        match &ast.node_type {
11063            ASTNodeType::Function { name, args } => {
11064                let normalized = name.to_uppercase();
11065                let mut spellings = vec![(String::new(), normalized.clone())];
11066                let mut stripped = normalized.as_str();
11067                while let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
11068                    .iter()
11069                    .find_map(|prefix| stripped.strip_prefix(prefix))
11070                {
11071                    stripped = rest;
11072                    spellings.push((String::new(), stripped.to_string()));
11073                }
11074                let resolved = crate::function_registry::resolve("", name);
11075                let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
11076                    || resolved.as_ref().is_some_and(|resolved| {
11077                        changed.contains(&(
11078                            resolved.namespace.clone(),
11079                            resolved.canonical_name.clone(),
11080                        ))
11081                    });
11082                directly_changed
11083                    || resolved.is_none()
11084                    || args
11085                        .iter()
11086                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
11087            }
11088            ASTNodeType::Call { callee, args } => {
11089                Self::ast_uses_changed_function(callee, changed)
11090                    || args
11091                        .iter()
11092                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
11093            }
11094            ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
11095            ASTNodeType::BinaryOp { left, right, .. } => {
11096                Self::ast_uses_changed_function(left, changed)
11097                    || Self::ast_uses_changed_function(right, changed)
11098            }
11099            ASTNodeType::Array(rows) => rows
11100                .iter()
11101                .flatten()
11102                .any(|node| Self::ast_uses_changed_function(node, changed)),
11103            _ => false,
11104        }
11105    }
11106
11107    fn ast_contains_function(ast: &ASTNode) -> bool {
11108        match &ast.node_type {
11109            ASTNodeType::Function { .. } => true,
11110            ASTNodeType::Call { callee, args } => {
11111                Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
11112            }
11113            ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
11114            ASTNodeType::BinaryOp { left, right, .. } => {
11115                Self::ast_contains_function(left) || Self::ast_contains_function(right)
11116            }
11117            ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
11118            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
11119        }
11120    }
11121
11122    fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
11123        Region::rows_from(sheet_id, start_row0)
11124    }
11125
11126    fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
11127        Region::cols_from(sheet_id, start_col0)
11128    }
11129
11130    #[cfg(test)]
11131    pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
11132        self.force_non_cycle_schedule_fallback_for_test = true;
11133    }
11134
11135    fn materialize_deferred_sheet_before_structural_edit(
11136        &mut self,
11137        sheet: &str,
11138    ) -> Result<(), crate::engine::EditorError> {
11139        if self.staged_formulas.contains_key(sheet) {
11140            self.build_graph_for_sheets([sheet])?;
11141        }
11142        Ok(())
11143    }
11144
11145    /// Refuse an insert of `count` rows (`rows`) or columns at 0-based
11146    /// `before0` that would push content off the grid, as Excel does; nothing
11147    /// is changed. Content is the Arrow extent (which may include trailing
11148    /// empty cells, so this can refuse conservatively) and every formula.
11149    fn check_insert_room(
11150        &self,
11151        sheet: &str,
11152        before0: u32,
11153        count: u32,
11154        rows: bool,
11155    ) -> Result<(), crate::engine::EditorError> {
11156        let limit = if rows {
11157            crate::engine::authority::geom::MAX_ROW
11158        } else {
11159            crate::engine::authority::geom::MAX_COL
11160        } + 1;
11161        let arrow = self.arrow_sheets.sheet(sheet).map_or(0, |a| {
11162            if rows {
11163                a.nrows
11164            } else {
11165                u32::try_from(a.columns.len()).unwrap_or(u32::MAX)
11166            }
11167        });
11168        let formulas = self.graph.sheet_id(sheet).map_or(0, |id| {
11169            self.graph
11170                .grid_vertices_in_sheet(id)
11171                .map(|(_, at)| if rows { at.row() } else { at.col() } + 1)
11172                .max()
11173                .unwrap_or(0)
11174        });
11175        let extent = arrow.max(formulas);
11176        let pushed_off = extent > before0 && extent.saturating_add(count) > limit;
11177        if count > limit || pushed_off {
11178            return Err(crate::engine::EditorError::Excel(
11179                ExcelError::new(ExcelErrorKind::Ref).with_message(format!(
11180                    "inserting {count} {} would push non-empty cells off the sheet",
11181                    if rows { "rows" } else { "columns" }
11182                )),
11183            ));
11184        }
11185        Ok(())
11186    }
11187
11188    fn structural_row_occupancy(
11189        &self,
11190        sheet: &str,
11191        sheet_id: SheetId,
11192    ) -> crate::engine::graph::StructuralOccupancy {
11193        if !self.graph.has_compressed_range_readers() {
11194            return crate::engine::graph::StructuralOccupancy::default();
11195        }
11196        let mut occupancy = self.graph.structural_occupancy(sheet_id);
11197        if let Some(arrow_sheet) = self.arrow_sheets.sheet(sheet) {
11198            occupancy.include_arrow_sheet(arrow_sheet);
11199            occupancy
11200        } else {
11201            // Missing Arrow state cannot prove an apparently empty column empty.
11202            crate::engine::graph::StructuralOccupancy::conservative()
11203        }
11204    }
11205
11206    fn structural_column_occupancy(&self) -> crate::engine::graph::StructuralOccupancy {
11207        // Arrow exposes occupied columns through chunk metadata and overlay maps,
11208        // but has no cheap occupied-row index. Column edits therefore deliberately
11209        // retain conservative cross-axis invalidation instead of scanning cells.
11210        crate::engine::graph::StructuralOccupancy::conservative()
11211    }
11212
11213    /// Insert rows (1-based) and mirror into Arrow store when enabled
11214    pub fn insert_rows(
11215        &mut self,
11216        sheet: &str,
11217        before: u32,
11218        count: u32,
11219    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11220    {
11221        if count == 0 {
11222            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11223        }
11224        self.observe_function_semantic_epoch()
11225            .map_err(crate::engine::EditorError::Excel)?;
11226        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11227        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11228        let sheet_id = self.ensure_known_sheet_id(sheet)?;
11229        let before0 = before.saturating_sub(1);
11230        self.check_insert_room(sheet, before0, count, true)?;
11231        let affected_region = Self::structural_row_region(sheet_id, before0);
11232        let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11233        let summary = {
11234            let mut editor =
11235                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11236            editor.insert_rows(sheet_id, before0, count)?
11237        };
11238        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11239            let before0 = before0 as usize;
11240            asheet.insert_rows(before0, count as usize);
11241        }
11242        self.purge_derived_formats_after_row(sheet_id, before0);
11243        self.mark_moved_formula_vertices_dirty(&summary);
11244        self.clear_computed_overlay_after_row(sheet, before0 as usize);
11245        self.shift_row_visibility_insert(sheet_id, before0, count);
11246        self.record_structural_change(StructuralScope::Region(affected_region));
11247        self.mark_topology_edited();
11248        Ok(summary)
11249    }
11250
11251    /// Delete rows (1-based) and mirror into Arrow store when enabled
11252    pub fn delete_rows(
11253        &mut self,
11254        sheet: &str,
11255        start: u32,
11256        count: u32,
11257    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11258    {
11259        if count == 0 {
11260            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11261        }
11262        self.observe_function_semantic_epoch()
11263            .map_err(crate::engine::EditorError::Excel)?;
11264        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11265        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11266        let sheet_id = self.ensure_known_sheet_id(sheet)?;
11267        let start0 = start.saturating_sub(1);
11268        let affected_region = Self::structural_row_region(sheet_id, start0);
11269        let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11270        let summary = {
11271            let mut editor =
11272                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11273            editor.delete_rows(sheet_id, start0, count)?
11274        };
11275        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11276            let start0 = start0 as usize;
11277            asheet.delete_rows(start0, count as usize);
11278        }
11279        self.purge_derived_formats_after_row(sheet_id, start0);
11280        self.mark_moved_formula_vertices_dirty(&summary);
11281        self.clear_computed_overlay_after_row(sheet, start0 as usize);
11282        self.shift_row_visibility_delete(sheet_id, start0, count);
11283        self.record_structural_change(StructuralScope::Region(affected_region));
11284        self.mark_topology_edited();
11285        Ok(summary)
11286    }
11287
11288    /// Insert columns (1-based) and mirror into Arrow store when enabled
11289    pub fn insert_columns(
11290        &mut self,
11291        sheet: &str,
11292        before: u32,
11293        count: u32,
11294    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11295    {
11296        if count == 0 {
11297            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11298        }
11299        self.observe_function_semantic_epoch()
11300            .map_err(crate::engine::EditorError::Excel)?;
11301        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11302        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11303        let sheet_id = self.graph.sheet_id(sheet).ok_or(
11304            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11305                name: sheet.to_string(),
11306                reason: "Unknown sheet".to_string(),
11307            },
11308        )?;
11309        let before0 = before.saturating_sub(1);
11310        self.check_insert_room(sheet, before0, count, false)?;
11311        let affected_region = Self::structural_col_region(sheet_id, before0);
11312        let occupancy = self.structural_column_occupancy();
11313        let summary = {
11314            let mut editor =
11315                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11316            editor.insert_columns(sheet_id, before0, count)?
11317        };
11318        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11319            let before0 = before0 as usize;
11320            asheet.insert_columns(before0, count as usize);
11321        }
11322        self.purge_derived_formats_after_col(sheet_id, before0);
11323        self.mark_moved_formula_vertices_dirty(&summary);
11324        self.clear_computed_overlay_after_col(sheet, before0 as usize);
11325        self.record_structural_change(StructuralScope::Region(affected_region));
11326        self.mark_topology_edited();
11327        Ok(summary)
11328    }
11329
11330    /// Delete columns (1-based) and mirror into Arrow store when enabled
11331    pub fn delete_columns(
11332        &mut self,
11333        sheet: &str,
11334        start: u32,
11335        count: u32,
11336    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11337    {
11338        if count == 0 {
11339            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11340        }
11341        self.observe_function_semantic_epoch()
11342            .map_err(crate::engine::EditorError::Excel)?;
11343        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11344        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11345        let sheet_id = self.graph.sheet_id(sheet).ok_or(
11346            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11347                name: sheet.to_string(),
11348                reason: "Unknown sheet".to_string(),
11349            },
11350        )?;
11351        let start0 = start.saturating_sub(1);
11352        let affected_region = Self::structural_col_region(sheet_id, start0);
11353        let occupancy = self.structural_column_occupancy();
11354        let summary = {
11355            let mut editor =
11356                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11357            editor.delete_columns(sheet_id, start0, count)?
11358        };
11359        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11360            let start0 = start0 as usize;
11361            asheet.delete_columns(start0, count as usize);
11362        }
11363        self.purge_derived_formats_after_col(sheet_id, start0);
11364        self.mark_moved_formula_vertices_dirty(&summary);
11365        self.clear_computed_overlay_after_col(sheet, start0 as usize);
11366        self.record_structural_change(StructuralScope::Region(affected_region));
11367        self.mark_topology_edited();
11368        Ok(summary)
11369    }
11370    /// Arrow-backed used row bounds across a column span (1-based inclusive cols).
11371    fn arrow_used_row_bounds(
11372        &self,
11373        sheet: &str,
11374        start_col: u32,
11375        end_col: u32,
11376    ) -> Option<(u32, u32)> {
11377        let a = self.sheet_store().sheet(sheet)?;
11378        if a.columns.is_empty() {
11379            return None;
11380        }
11381        let sc0 = start_col.saturating_sub(1) as usize;
11382        let ec0 = end_col.saturating_sub(1) as usize;
11383        let col_hi = a.columns.len().saturating_sub(1);
11384        if sc0 > col_hi {
11385            return None;
11386        }
11387        let ec0 = ec0.min(col_hi);
11388        // Pass-scoped cache with snapshot guard
11389        let snap = self.data_snapshot_id();
11390        let mut min_r0: Option<usize> = None;
11391        for ci in sc0..=ec0 {
11392            let sheet_id = self.graph.sheet_id(sheet)?;
11393            if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
11394                g.as_ref()
11395                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11396            }) {
11397                let mv = mv as usize;
11398                min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
11399                continue;
11400            }
11401            // Compute and store
11402            let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11403            if let Ok(mut g) = self.row_bounds_cache.write() {
11404                g.get_or_insert_with(|| RowBoundsCache::new(snap))
11405                    .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
11406            }
11407            if let Some(m) = min_c {
11408                min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
11409            }
11410        }
11411        min_r0?;
11412        let mut max_r0: Option<usize> = None;
11413        for ci in sc0..=ec0 {
11414            let sheet_id = self.graph.sheet_id(sheet)?;
11415            if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
11416                g.as_ref()
11417                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11418            }) {
11419                let mv = mv as usize;
11420                max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
11421                continue;
11422            }
11423            let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11424            if let Ok(mut g) = self.row_bounds_cache.write() {
11425                g.get_or_insert_with(|| RowBoundsCache::new(snap))
11426                    .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
11427            }
11428            if let Some(m) = max_c {
11429                max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
11430            }
11431        }
11432        match (min_r0, max_r0) {
11433            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11434            _ => None,
11435        }
11436    }
11437
11438    fn scan_column_used_bounds(
11439        a: &crate::arrow_store::ArrowSheet,
11440        ci: usize,
11441    ) -> (Option<u32>, Option<u32>) {
11442        let col = &a.columns[ci];
11443
11444        // Min: scan dense chunks first, then sparse chunks in ascending index order.
11445        let mut min_r0: Option<u32> = None;
11446        for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11447            let tags = chunk.type_tag.values();
11448            for (off, &t) in tags.iter().enumerate() {
11449                let overlay_non_empty = chunk
11450                    .overlay
11451                    .get(off)
11452                    .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11453                    .unwrap_or(false)
11454                    || chunk
11455                        .computed_overlay
11456                        .get(off)
11457                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11458                        .unwrap_or(false);
11459                if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11460                    let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11461                        break;
11462                    };
11463                    let row0 = chunk_start + off;
11464                    min_r0 = Some(row0 as u32);
11465                    break;
11466                }
11467            }
11468            if min_r0.is_some() {
11469                break;
11470            }
11471        }
11472        if min_r0.is_none() && !col.sparse_chunks.is_empty() {
11473            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11474            sparse_idxs.sort_unstable();
11475            for chunk_idx in sparse_idxs {
11476                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11477                    continue;
11478                };
11479                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11480                    continue;
11481                };
11482                let tags = chunk.type_tag.values();
11483                for (off, &t) in tags.iter().enumerate() {
11484                    let overlay_non_empty = chunk
11485                        .overlay
11486                        .get(off)
11487                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11488                        .unwrap_or(false)
11489                        || chunk
11490                            .computed_overlay
11491                            .get(off)
11492                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11493                            .unwrap_or(false);
11494                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11495                        let row0 = chunk_start + off;
11496                        min_r0 = Some(row0 as u32);
11497                        break;
11498                    }
11499                }
11500                if min_r0.is_some() {
11501                    break;
11502                }
11503            }
11504        }
11505
11506        // Max: scan sparse chunks in descending index order, then dense chunks in reverse.
11507        let mut max_r0: Option<u32> = None;
11508        if !col.sparse_chunks.is_empty() {
11509            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11510            sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
11511            for chunk_idx in sparse_idxs {
11512                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11513                    continue;
11514                };
11515                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11516                    continue;
11517                };
11518                let tags = chunk.type_tag.values();
11519                for (rev_idx, &t) in tags.iter().enumerate().rev() {
11520                    let overlay_non_empty = chunk
11521                        .overlay
11522                        .get(rev_idx)
11523                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11524                        .unwrap_or(false)
11525                        || chunk
11526                            .computed_overlay
11527                            .get(rev_idx)
11528                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11529                            .unwrap_or(false);
11530                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11531                        let row0 = chunk_start + rev_idx;
11532                        max_r0 = Some(row0 as u32);
11533                        break;
11534                    }
11535                }
11536                if max_r0.is_some() {
11537                    break;
11538                }
11539            }
11540        }
11541        if max_r0.is_none() {
11542            for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
11543                let tags = chunk.type_tag.values();
11544                for (rev_idx, &t) in tags.iter().enumerate().rev() {
11545                    let overlay_non_empty = chunk
11546                        .overlay
11547                        .get(rev_idx)
11548                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11549                        .unwrap_or(false)
11550                        || chunk
11551                            .computed_overlay
11552                            .get(rev_idx)
11553                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11554                            .unwrap_or(false);
11555                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11556                        let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11557                            break;
11558                        };
11559                        let row0 = chunk_start + rev_idx;
11560                        max_r0 = Some(row0 as u32);
11561                        break;
11562                    }
11563                }
11564                if max_r0.is_some() {
11565                    break;
11566                }
11567            }
11568        }
11569
11570        (min_r0, max_r0)
11571    }
11572
11573    /// Arrow-backed used column bounds across a row span (1-based inclusive rows).
11574    fn arrow_used_col_bounds(
11575        &self,
11576        sheet: &str,
11577        start_row: u32,
11578        end_row: u32,
11579    ) -> Option<(u32, u32)> {
11580        let a = self.sheet_store().sheet(sheet)?;
11581        if a.columns.is_empty() {
11582            return None;
11583        }
11584        let sr0 = start_row.saturating_sub(1) as usize;
11585        let er0 = end_row.saturating_sub(1) as usize;
11586        if sr0 > er0 {
11587            return None;
11588        }
11589        // Map start/end rows into chunk ranges
11590        // We will scan each column for any non-empty within [sr0..=er0]
11591        let mut min_c0: Option<usize> = None;
11592        let mut max_c0: Option<usize> = None;
11593        // Precompute chunk bounds for row range
11594        for (ci, col) in a.columns.iter().enumerate() {
11595            let mut any_in_range = false;
11596
11597            let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
11598                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11599                    return false;
11600                };
11601                let chunk_len = chunk.type_tag.len();
11602                if chunk_len == 0 {
11603                    return false;
11604                }
11605                let chunk_end = chunk_start + chunk_len.saturating_sub(1);
11606                // check intersection
11607                if sr0 > chunk_end || er0 < chunk_start {
11608                    return false;
11609                }
11610                let start_off = sr0.max(chunk_start) - chunk_start;
11611                let end_off = er0.min(chunk_end) - chunk_start;
11612                let tags = chunk.type_tag.values();
11613                for off in start_off..=end_off {
11614                    let overlay_non_empty = chunk
11615                        .overlay
11616                        .get(off)
11617                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11618                        .unwrap_or(false)
11619                        || chunk
11620                            .computed_overlay
11621                            .get(off)
11622                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11623                            .unwrap_or(false);
11624                    if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
11625                        return true;
11626                    }
11627                }
11628                false
11629            };
11630
11631            for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11632                if scan_chunk(chunk_idx, chunk) {
11633                    any_in_range = true;
11634                    break;
11635                }
11636            }
11637
11638            if !any_in_range && !col.sparse_chunks.is_empty() {
11639                for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
11640                    if scan_chunk(chunk_idx, chunk) {
11641                        any_in_range = true;
11642                        break;
11643                    }
11644                }
11645            }
11646
11647            if any_in_range {
11648                min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
11649                max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
11650            }
11651        }
11652        match (min_c0, max_c0) {
11653            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11654            _ => None,
11655        }
11656    }
11657
11658    fn formula_row_bounds_for_columns(
11659        &self,
11660        sheet: &str,
11661        start_col: u32,
11662        end_col: u32,
11663    ) -> Option<(u32, u32)> {
11664        let sheet_id = self.graph.sheet_id(sheet)?;
11665        let sc0 = start_col.saturating_sub(1);
11666        let ec0 = end_col.saturating_sub(1);
11667        let mut min_r0: Option<u32> = None;
11668        let mut max_r0: Option<u32> = None;
11669
11670        if self.graph.sheet_index(sheet_id).is_some() {
11671            for vid in self.graph.vertices_in_cols(sheet_id, sc0, ec0) {
11672                if !matches!(
11673                    self.graph.get_vertex_kind(vid),
11674                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11675                ) {
11676                    continue;
11677                }
11678                let Some(row0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.row()) else {
11679                    continue;
11680                };
11681                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11682                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11683            }
11684        } else {
11685            for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11686                if !matches!(
11687                    self.graph.get_vertex_kind(vid),
11688                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11689                ) {
11690                    continue;
11691                }
11692                let col0 = coord.col();
11693                if col0 < sc0 || col0 > ec0 {
11694                    continue;
11695                }
11696                let row0 = coord.row();
11697                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11698                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11699            }
11700        }
11701
11702        match (min_r0, max_r0) {
11703            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11704            _ => None,
11705        }
11706    }
11707
11708    fn formula_col_bounds_for_rows(
11709        &self,
11710        sheet: &str,
11711        start_row: u32,
11712        end_row: u32,
11713    ) -> Option<(u32, u32)> {
11714        let sheet_id = self.graph.sheet_id(sheet)?;
11715        let sr0 = start_row.saturating_sub(1);
11716        let er0 = end_row.saturating_sub(1);
11717        let mut min_c0: Option<u32> = None;
11718        let mut max_c0: Option<u32> = None;
11719
11720        if self.graph.sheet_index(sheet_id).is_some() {
11721            for vid in self.graph.vertices_in_rows(sheet_id, sr0, er0) {
11722                if !matches!(
11723                    self.graph.get_vertex_kind(vid),
11724                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11725                ) {
11726                    continue;
11727                }
11728                let Some(col0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.col()) else {
11729                    continue;
11730                };
11731                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11732                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11733            }
11734        } else {
11735            for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11736                if !matches!(
11737                    self.graph.get_vertex_kind(vid),
11738                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11739                ) {
11740                    continue;
11741                }
11742                let row0 = coord.row();
11743                if row0 < sr0 || row0 > er0 {
11744                    continue;
11745                }
11746                let col0 = coord.col();
11747                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11748                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11749            }
11750        }
11751
11752        match (min_c0, max_c0) {
11753            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11754            _ => None,
11755        }
11756    }
11757
11758    fn union_used_bounds(
11759        first: Option<(u32, u32)>,
11760        second: Option<(u32, u32)>,
11761    ) -> Option<(u32, u32)> {
11762        match (first, second) {
11763            (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
11764            (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
11765            (None, None) => None,
11766        }
11767    }
11768
11769    /// Mirror a single cell value into the Arrow overlay if enabled.
11770    /// Handles capacity growth, per-chunk overlay set, and heuristic compaction.
11771    fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
11772        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11773            return;
11774        }
11775        if self.arrow_sheets.sheet(sheet).is_none() {
11776            self.arrow_sheets
11777                .sheets
11778                .push(crate::arrow_store::ArrowSheet {
11779                    name: std::sync::Arc::<str>::from(sheet),
11780                    date_system: self.config.date_system,
11781                    columns: Vec::new(),
11782                    nrows: 0,
11783                    chunk_starts: Vec::new(),
11784                    chunk_rows: 32 * 1024,
11785                });
11786        }
11787
11788        let row0 = row.saturating_sub(1) as usize;
11789        let col0 = col.saturating_sub(1) as usize;
11790
11791        let asheet = self
11792            .arrow_sheets
11793            .sheet_mut(sheet)
11794            .expect("ArrowSheet must exist");
11795
11796        let cur_cols = asheet.columns.len();
11797        if col0 >= cur_cols {
11798            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11799        }
11800
11801        if row0 >= asheet.nrows as usize {
11802            if asheet.columns.is_empty() {
11803                asheet.insert_columns(0, 1);
11804            }
11805            asheet.ensure_row_capacity(row0 + 1);
11806        }
11807        if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
11808            let ov =
11809                crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
11810            let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
11811                let _ = ch.overlay.set(in_off, ov);
11812                let format = match value {
11813                    LiteralValue::Date(_) => Some(crate::format::FormatId::DATE),
11814                    LiteralValue::DateTime(_) => Some(crate::format::FormatId::DATETIME),
11815                    LiteralValue::Time(_) => Some(crate::format::FormatId::TIME),
11816                    LiteralValue::Duration(_) => Some(crate::format::FormatId::DURATION),
11817                    _ => None,
11818                };
11819                ch.overlay.set_format(in_off, format);
11820                // A user edit must invalidate any computed (formula/spill) overlay entry at
11821                // this cell. Otherwise, if the delta overlay later compacts into the base lanes
11822                // (clearing `overlay`), a stale `computed_overlay=Empty` could incorrectly mask
11823                // the edited base value under the read cascade.
11824                ch.computed_overlay.remove(in_off)
11825            } else {
11826                return;
11827            };
11828            // Heuristic compaction: > len/50 or > 1024
11829            let abs_threshold = 1024usize;
11830            let frac_den = 50usize;
11831            let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
11832            if freed > 0 {
11833                self.overlay_compactions = self.overlay_compactions.saturating_add(1);
11834            }
11835            self.adjust_computed_overlay_bytes(computed_delta);
11836        }
11837    }
11838
11839    /// Remove a delta-overlay entry for a single cell (if present).
11840    ///
11841    /// This is used when transitioning a cell to a formula so that any previous user-edit overlay
11842    /// does not continue to mask computed overlay outputs.
11843    fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
11844        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11845            return;
11846        }
11847        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11848            return;
11849        };
11850        let row0 = row.saturating_sub(1) as usize;
11851        let col0 = col.saturating_sub(1) as usize;
11852        if row0 >= asheet.nrows as usize {
11853            return;
11854        }
11855        if col0 >= asheet.columns.len() {
11856            return;
11857        }
11858        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11859            return;
11860        };
11861        if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
11862            let _ = ch.overlay.remove(in_off);
11863        }
11864    }
11865
11866    fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
11867        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11868            return;
11869        }
11870
11871        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11872            return;
11873        };
11874        if start_row0 >= asheet.nrows as usize {
11875            return;
11876        }
11877
11878        let starts = asheet.chunk_starts.clone();
11879        let nrows = asheet.nrows as usize;
11880        let mut delta = 0isize;
11881        for col in &mut asheet.columns {
11882            for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
11883                let Some(&chunk_start) = starts.get(chunk_idx) else {
11884                    continue;
11885                };
11886                let chunk_end = starts
11887                    .get(chunk_idx + 1)
11888                    .copied()
11889                    .unwrap_or(nrows)
11890                    .min(chunk_start.saturating_add(ch.len()));
11891                if chunk_end <= start_row0 {
11892                    continue;
11893                }
11894                if chunk_start >= start_row0 {
11895                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11896                } else {
11897                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11898                    delta = delta
11899                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11900                }
11901            }
11902
11903            for (chunk_idx, ch) in &mut col.sparse_chunks {
11904                let Some(&chunk_start) = starts.get(*chunk_idx) else {
11905                    continue;
11906                };
11907                let chunk_end = starts
11908                    .get(*chunk_idx + 1)
11909                    .copied()
11910                    .unwrap_or(nrows)
11911                    .min(chunk_start.saturating_add(ch.len()));
11912                if chunk_end <= start_row0 {
11913                    continue;
11914                }
11915                if chunk_start >= start_row0 {
11916                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11917                } else {
11918                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11919                    delta = delta
11920                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11921                }
11922            }
11923        }
11924        self.adjust_computed_overlay_bytes(delta);
11925    }
11926
11927    fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
11928        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11929            return;
11930        }
11931
11932        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11933            return;
11934        };
11935        if start_col0 >= asheet.columns.len() {
11936            return;
11937        }
11938
11939        let mut delta = 0isize;
11940        for col in asheet.columns.iter_mut().skip(start_col0) {
11941            for ch in &mut col.chunks {
11942                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11943            }
11944            for ch in col.sparse_chunks.values_mut() {
11945                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11946            }
11947        }
11948        self.adjust_computed_overlay_bytes(delta);
11949    }
11950
11951    #[inline]
11952    fn literal_to_overlay_value(
11953        value: &LiteralValue,
11954        date_system: crate::engine::DateSystem,
11955    ) -> crate::arrow_store::OverlayValue {
11956        crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
11957    }
11958
11959    fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
11960        self.arrow_sheets
11961            .sheet(sheet)
11962            .map(|sheet| sheet.date_system)
11963            .unwrap_or(self.config.date_system)
11964    }
11965
11966    /// Read a single cell's delta overlay entry (if present), preserving the distinction between
11967    /// absent and explicit `Empty`.
11968    fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11969        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11970            return None;
11971        }
11972        let asheet = self.arrow_sheets.sheet(sheet)?;
11973        let row0 = row.saturating_sub(1) as usize;
11974        let col0 = col.saturating_sub(1) as usize;
11975        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11976            return None;
11977        }
11978        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
11979        let ch = asheet.columns[col0].chunk(ch_idx)?;
11980        ch.overlay
11981            .get_scalar(in_off)
11982            .map(|ov| ov.to_literal_for(asheet.date_system))
11983    }
11984
11985    /// Read a single cell's computed overlay entry (if present), preserving the distinction
11986    /// between absent and explicit `Empty`.
11987    fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11988        if !(self.config.arrow_storage_enabled
11989            && self.config.delta_overlay_enabled
11990            && self.config.write_formula_overlay_enabled)
11991        {
11992            return None;
11993        }
11994        let asheet = self.arrow_sheets.sheet(sheet)?;
11995        let row0 = row.saturating_sub(1) as usize;
11996        let col0 = col.saturating_sub(1) as usize;
11997        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11998            return None;
11999        }
12000        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
12001        let ch = asheet.columns[col0].chunk(ch_idx)?;
12002        ch.computed_overlay
12003            .get_scalar(in_off)
12004            .map(|ov| ov.to_literal_for(asheet.date_system))
12005    }
12006
12007    fn set_delta_overlay_cell_raw(
12008        &mut self,
12009        sheet: &str,
12010        row: u32,
12011        col: u32,
12012        value: Option<LiteralValue>,
12013    ) {
12014        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
12015            return;
12016        }
12017
12018        self.ensure_arrow_sheet(sheet);
12019        let date_system = self.arrow_sheet_date_system(sheet);
12020        let ov_opt = value
12021            .as_ref()
12022            .map(|value| Self::literal_to_overlay_value(value, date_system));
12023        let row0 = row.saturating_sub(1) as usize;
12024        let col0 = col.saturating_sub(1) as usize;
12025        let asheet = self
12026            .arrow_sheets
12027            .sheet_mut(sheet)
12028            .expect("ArrowSheet must exist");
12029
12030        let cur_cols = asheet.columns.len();
12031        if col0 >= cur_cols {
12032            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12033        }
12034        if row0 >= asheet.nrows as usize {
12035            if asheet.columns.is_empty() {
12036                asheet.insert_columns(0, 1);
12037            }
12038            asheet.ensure_row_capacity(row0 + 1);
12039        }
12040
12041        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12042            return;
12043        };
12044        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12045            return;
12046        };
12047
12048        if let Some(ov) = ov_opt {
12049            let _ = ch.overlay.set(in_off, ov);
12050        } else {
12051            let _ = ch.overlay.remove(in_off);
12052        }
12053    }
12054
12055    fn set_computed_overlay_cell_raw(
12056        &mut self,
12057        sheet: &str,
12058        row: u32,
12059        col: u32,
12060        value: Option<LiteralValue>,
12061    ) {
12062        if !(self.config.arrow_storage_enabled
12063            && self.config.delta_overlay_enabled
12064            && self.config.write_formula_overlay_enabled)
12065        {
12066            return;
12067        }
12068
12069        self.ensure_arrow_sheet(sheet);
12070        let date_system = self.arrow_sheet_date_system(sheet);
12071        let ov_opt = value
12072            .as_ref()
12073            .map(|value| Self::literal_to_overlay_value(value, date_system));
12074        let row0 = row.saturating_sub(1) as usize;
12075        let col0 = col.saturating_sub(1) as usize;
12076        let asheet = self
12077            .arrow_sheets
12078            .sheet_mut(sheet)
12079            .expect("ArrowSheet must exist");
12080
12081        let cur_cols = asheet.columns.len();
12082        if col0 >= cur_cols {
12083            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12084        }
12085        if row0 >= asheet.nrows as usize {
12086            if asheet.columns.is_empty() {
12087                asheet.insert_columns(0, 1);
12088            }
12089            asheet.ensure_row_capacity(row0 + 1);
12090        }
12091
12092        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12093            return;
12094        };
12095        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12096            return;
12097        };
12098
12099        let delta = if let Some(ov) = ov_opt {
12100            ch.computed_overlay.set(in_off, ov)
12101        } else {
12102            ch.computed_overlay.remove(in_off)
12103        };
12104        self.adjust_computed_overlay_bytes(delta);
12105    }
12106
12107    fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
12108        use crate::engine::ArrowOp;
12109
12110        let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
12111            Box::new(batch.ops.iter().rev())
12112        } else {
12113            Box::new(batch.ops.iter())
12114        };
12115
12116        for op in iter {
12117            match op {
12118                ArrowOp::SetDeltaCell {
12119                    sheet_id,
12120                    row0,
12121                    col0,
12122                    old,
12123                    new,
12124                } => {
12125                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12126                    let v = if undo { old.clone() } else { new.clone() };
12127                    self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
12128                }
12129                ArrowOp::SetComputedCell {
12130                    sheet_id,
12131                    row0,
12132                    col0,
12133                    old,
12134                    new,
12135                } => {
12136                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12137                    let v = if undo { old.clone() } else { new.clone() };
12138                    self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
12139                }
12140                ArrowOp::RestoreComputedRect {
12141                    sheet_id,
12142                    sr0,
12143                    sc0,
12144                    er0,
12145                    ec0,
12146                    old,
12147                    new,
12148                } => {
12149                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12150                    let vals = if undo { old } else { new };
12151                    let height = (*er0).saturating_sub(*sr0) as usize + 1;
12152                    let width = (*ec0).saturating_sub(*sc0) as usize + 1;
12153                    for r in 0..height {
12154                        for c in 0..width {
12155                            let v = vals
12156                                .get(r)
12157                                .and_then(|row| row.get(c))
12158                                .cloned()
12159                                .unwrap_or(LiteralValue::Empty);
12160                            self.set_computed_overlay_cell_raw(
12161                                &sheet,
12162                                *sr0 + 1 + r as u32,
12163                                *sc0 + 1 + c as u32,
12164                                Some(v),
12165                            );
12166                        }
12167                    }
12168                }
12169                ArrowOp::InsertRows {
12170                    sheet_id,
12171                    before0,
12172                    count,
12173                } => {
12174                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12175                    self.ensure_arrow_sheet(&sheet);
12176                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
12177                        if undo {
12178                            asheet.delete_rows(*before0 as usize, *count as usize);
12179                        } else {
12180                            asheet.insert_rows(*before0 as usize, *count as usize);
12181                        }
12182                    }
12183                    self.purge_derived_formats_after_row(*sheet_id, *before0);
12184                }
12185                ArrowOp::InsertCols {
12186                    sheet_id,
12187                    before0,
12188                    count,
12189                } => {
12190                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12191                    self.ensure_arrow_sheet(&sheet);
12192                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
12193                        if undo {
12194                            asheet.delete_columns(*before0 as usize, *count as usize);
12195                        } else {
12196                            asheet.insert_columns(*before0 as usize, *count as usize);
12197                        }
12198                    }
12199                    self.purge_derived_formats_after_col(*sheet_id, *before0);
12200                }
12201            }
12202        }
12203    }
12204
12205    fn record_spill_ops_into_arrow_undo(
12206        &mut self,
12207        undo: &mut crate::engine::ArrowUndoBatch,
12208        events: &[crate::engine::ChangeEvent],
12209    ) {
12210        use crate::engine::ChangeEvent;
12211        use formualizer_common::LiteralValue;
12212
12213        #[allow(clippy::type_complexity)]
12214        let rect_from_snapshot =
12215            |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
12216             -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
12217                if snap.target_cells.is_empty() {
12218                    return None;
12219                }
12220                let sheet_id = snap.target_cells[0].sheet_id;
12221                let sr0 = snap.target_cells[0].coord.row();
12222                let sc0 = snap.target_cells[0].coord.col();
12223                if snap.values.is_empty() || snap.values[0].is_empty() {
12224                    return None;
12225                }
12226                let h = snap.values.len() as u32;
12227                let w = snap.values[0].len() as u32;
12228                let er0 = sr0.saturating_add(h.saturating_sub(1));
12229                let ec0 = sc0.saturating_add(w.saturating_sub(1));
12230                Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
12231            };
12232
12233        for ev in events {
12234            match ev {
12235                ChangeEvent::SpillCommitted { old, new, .. } => {
12236                    if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
12237                        let old_vals = if let Some(old_snap) = old {
12238                            rect_from_snapshot(old_snap)
12239                                .map(|(_, _, _, _, _, v)| v)
12240                                .unwrap_or_else(|| {
12241                                    vec![
12242                                        vec![LiteralValue::Empty; new_vals[0].len()];
12243                                        new_vals.len()
12244                                    ]
12245                                })
12246                        } else {
12247                            vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
12248                        };
12249                        undo.record_restore_computed_rect(
12250                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
12251                        );
12252                    }
12253                }
12254                ChangeEvent::SpillCleared { old, .. } => {
12255                    if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
12256                        let new_vals =
12257                            vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
12258                        undo.record_restore_computed_rect(
12259                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
12260                        );
12261                    }
12262                }
12263                _ => {}
12264            }
12265        }
12266    }
12267
12268    /// Mirror a value into the computed overlay (formula/spill outputs).
12269    ///
12270    /// This path is subject to `EvalConfig.max_overlay_memory_bytes`.
12271    /// If the cap is exceeded, computed overlays are compacted into base lanes.
12272    fn mirror_value_to_computed_overlay(
12273        &mut self,
12274        sheet: &str,
12275        row: u32,
12276        col: u32,
12277        value: &LiteralValue,
12278    ) {
12279        if !(self.config.arrow_storage_enabled
12280            && self.config.delta_overlay_enabled
12281            && self.config.write_formula_overlay_enabled)
12282        {
12283            return;
12284        }
12285        if self.computed_overlay_mirroring_disabled {
12286            return;
12287        }
12288
12289        let date_system = self.arrow_sheet_date_system(sheet);
12290        let ov = Self::literal_to_overlay_value(value, date_system);
12291        self.write_computed_overlay_value_0based(
12292            sheet,
12293            row.saturating_sub(1),
12294            col.saturating_sub(1),
12295            ov,
12296        );
12297    }
12298
12299    fn record_derived_format(&self, vertex_id: VertexId, format: Option<crate::format::FormatId>) {
12300        if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
12301            self.record_derived_format_at(cell, format);
12302        }
12303    }
12304
12305    fn record_derived_format_at(&self, cell: CellRef, format: Option<crate::format::FormatId>) {
12306        #[cfg(test)]
12307        self.derived_format_operations_for_test
12308            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12309        let format = format.filter(|id| *id != crate::format::FormatId::GENERAL);
12310        self.derived_formats.set(cell, format);
12311    }
12312
12313    fn clear_cell_format_state(&mut self, sheet: &str, cell: CellRef) {
12314        self.derived_formats.set(cell, None);
12315        if let Some(arrow) = self.arrow_sheets.sheet_mut(sheet) {
12316            arrow.clear_format(cell.coord.row() as usize, cell.coord.col() as usize);
12317        }
12318    }
12319
12320    fn clear_logged_cell_format_states(&mut self, events: &[ChangeEvent]) {
12321        let cells = events
12322            .iter()
12323            .filter_map(|event| match event {
12324                ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
12325                    Some(*addr)
12326                }
12327                _ => None,
12328            })
12329            .collect::<FxHashSet<_>>();
12330        for cell in cells {
12331            let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12332            self.clear_cell_format_state(&sheet, cell);
12333        }
12334    }
12335
12336    fn purge_derived_formats_after_row(&mut self, sheet_id: SheetId, start0: u32) {
12337        self.derived_formats
12338            .retain(|cell| cell.sheet_id != sheet_id || cell.coord.row() < start0);
12339    }
12340
12341    fn purge_derived_formats_after_col(&mut self, sheet_id: SheetId, start0: u32) {
12342        self.derived_formats
12343            .retain(|cell| cell.sheet_id != sheet_id || cell.coord.col() < start0);
12344    }
12345
12346    fn purge_derived_formats_for_sheet(&mut self, sheet_id: SheetId) {
12347        self.derived_formats
12348            .retain(|cell| cell.sheet_id != sheet_id);
12349    }
12350
12351    #[cfg(test)]
12352    pub(crate) fn debug_computed_overlay_format_0based(
12353        &self,
12354        sheet: &str,
12355        row0: u32,
12356        col0: u32,
12357    ) -> Option<crate::format::FormatId> {
12358        let sheet = self.arrow_sheets.sheet(sheet)?;
12359        let (chunk_idx, row_in_chunk) = sheet.chunk_of_row(row0 as usize)?;
12360        sheet
12361            .columns
12362            .get(col0 as usize)?
12363            .chunk(chunk_idx)?
12364            .computed_overlay
12365            .get_format(row_in_chunk)
12366    }
12367
12368    #[cfg(test)]
12369    pub(crate) fn debug_computed_overlay_chunk_has_formats_0based(
12370        &self,
12371        sheet: &str,
12372        row0: u32,
12373        col0: u32,
12374    ) -> bool {
12375        let Some(sheet) = self.arrow_sheets.sheet(sheet) else {
12376            return false;
12377        };
12378        let Some((chunk_idx, _)) = sheet.chunk_of_row(row0 as usize) else {
12379            return false;
12380        };
12381        sheet
12382            .columns
12383            .get(col0 as usize)
12384            .and_then(|column| column.chunk(chunk_idx))
12385            .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12386    }
12387
12388    #[cfg(test)]
12389    pub(crate) fn debug_clear_derived_format_0based(&mut self, sheet: &str, row0: u32, col0: u32) {
12390        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12391            self.derived_formats
12392                .set(CellRef::new_absolute(sheet_id, row0, col0), None);
12393        }
12394    }
12395
12396    #[cfg(test)]
12397    pub(crate) fn debug_record_derived_format_0based(
12398        &self,
12399        sheet: &str,
12400        row0: u32,
12401        col0: u32,
12402        format: Option<crate::format::FormatId>,
12403    ) {
12404        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12405            self.record_derived_format_at(CellRef::new_absolute(sheet_id, row0, col0), format);
12406        }
12407    }
12408
12409    #[cfg(test)]
12410    pub(crate) fn debug_derived_format_0based(
12411        &self,
12412        sheet: &str,
12413        row0: u32,
12414        col0: u32,
12415    ) -> Option<crate::format::FormatId> {
12416        let sheet_id = self.graph.sheet_id(sheet)?;
12417        self.derived_formats
12418            .get(&CellRef::new_absolute(sheet_id, row0, col0))
12419    }
12420
12421    #[cfg(test)]
12422    pub(crate) fn debug_reset_format_write_operation_counts(&mut self) {
12423        self.derived_format_operations_for_test
12424            .store(0, std::sync::atomic::Ordering::Relaxed);
12425        self.computed_overlay_set_explicit_entry_operations_for_test = 0;
12426        self.computed_overlay_stale_clear_range_effects_for_test = 0;
12427        self.computed_overlay_stale_clear_offset_attempts_for_test = 0;
12428        self.computed_format_vector_allocations_for_test
12429            .store(0, std::sync::atomic::Ordering::Relaxed);
12430    }
12431
12432    #[cfg(test)]
12433    pub(crate) fn debug_format_write_operation_counts(&self) -> (u64, u64, u64, u64, u64) {
12434        (
12435            self.derived_format_operations_for_test
12436                .load(std::sync::atomic::Ordering::Relaxed),
12437            self.computed_overlay_set_explicit_entry_operations_for_test,
12438            self.computed_format_vector_allocations_for_test
12439                .load(std::sync::atomic::Ordering::Relaxed),
12440            self.computed_overlay_stale_clear_range_effects_for_test,
12441            self.computed_overlay_stale_clear_offset_attempts_for_test,
12442        )
12443    }
12444
12445    fn write_computed_overlay_format_0based(
12446        &mut self,
12447        sheet: &str,
12448        row0: u32,
12449        col0: u32,
12450        format: Option<crate::format::FormatId>,
12451    ) {
12452        self.ensure_arrow_sheet(sheet);
12453        let (row0, col0) = (row0 as usize, col0 as usize);
12454        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
12455            return;
12456        };
12457        if col0 >= asheet.columns.len() {
12458            asheet.insert_columns(asheet.columns.len(), col0 + 1 - asheet.columns.len());
12459        }
12460        if row0 >= asheet.nrows as usize {
12461            asheet.ensure_row_capacity(row0 + 1);
12462        }
12463        let Some((chunk, offset)) = asheet.chunk_of_row(row0) else {
12464            return;
12465        };
12466        if let Some(chunk) = asheet.ensure_column_chunk_mut(col0, chunk) {
12467            chunk.computed_overlay.set_format(offset, format);
12468        }
12469    }
12470
12471    /// One unbuffered computed write of `value` and its derived `format` at
12472    /// `cell` (a single sheet lookup; the same writes as
12473    /// `write_computed_overlay_value_0based` then
12474    /// `write_computed_overlay_format_0based`).
12475    fn write_computed_cell_0based(
12476        &mut self,
12477        cell: CellRef,
12478        value: &LiteralValue,
12479        format: Option<crate::format::FormatId>,
12480    ) {
12481        if !(self.config.arrow_storage_enabled
12482            && self.config.delta_overlay_enabled
12483            && self.config.write_formula_overlay_enabled)
12484            || self.computed_overlay_mirroring_disabled
12485        {
12486            return;
12487        }
12488        if self.arrow_sheet_by_id(cell.sheet_id).is_none() {
12489            let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12490            self.ensure_arrow_sheet(&sheet);
12491        }
12492        let (row0, col0) = (cell.coord.row() as usize, cell.coord.col() as usize);
12493        let Some(asheet) = self.arrow_sheet_by_id_mut(cell.sheet_id) else {
12494            return;
12495        };
12496        let ov = Self::literal_to_overlay_value(value, asheet.date_system);
12497        let cur_cols = asheet.columns.len();
12498        if col0 >= cur_cols {
12499            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12500        }
12501        if row0 >= asheet.nrows as usize {
12502            asheet.ensure_row_capacity(row0 + 1);
12503        }
12504        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12505            return;
12506        };
12507        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12508            return;
12509        };
12510        let delta = ch.computed_overlay.set_scalar(in_off, ov);
12511        ch.computed_overlay.set_format(in_off, format);
12512        self.adjust_computed_overlay_bytes(delta);
12513        if let Some(cap) = self.config.max_overlay_memory_bytes
12514            && self.computed_overlay_bytes_estimate > cap
12515        {
12516            self.disable_computed_overlay_mirroring_due_to_budget(cap);
12517        }
12518    }
12519
12520    fn write_computed_overlay_value_0based(
12521        &mut self,
12522        sheet: &str,
12523        row0: u32,
12524        col0: u32,
12525        value: OverlayValue,
12526    ) {
12527        if !(self.config.arrow_storage_enabled
12528            && self.config.delta_overlay_enabled
12529            && self.config.write_formula_overlay_enabled)
12530        {
12531            return;
12532        }
12533        if self.computed_overlay_mirroring_disabled {
12534            return;
12535        }
12536
12537        self.ensure_arrow_sheet(sheet);
12538
12539        let row0 = row0 as usize;
12540        let col0 = col0 as usize;
12541        let asheet = self
12542            .arrow_sheets
12543            .sheet_mut(sheet)
12544            .expect("ArrowSheet must exist");
12545
12546        let cur_cols = asheet.columns.len();
12547        if col0 >= cur_cols {
12548            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12549        }
12550
12551        if row0 >= asheet.nrows as usize {
12552            if asheet.columns.is_empty() {
12553                asheet.insert_columns(0, 1);
12554            }
12555            asheet.ensure_row_capacity(row0 + 1);
12556        }
12557
12558        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12559            return;
12560        };
12561        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12562            return;
12563        };
12564
12565        let delta = ch.computed_overlay.set_scalar(in_off, value);
12566        self.adjust_computed_overlay_bytes(delta);
12567
12568        if let Some(cap) = self.config.max_overlay_memory_bytes
12569            && self.computed_overlay_bytes_estimate > cap
12570        {
12571            self.disable_computed_overlay_mirroring_due_to_budget(cap);
12572        }
12573    }
12574
12575    pub(crate) fn plan_computed_write_coalescing(
12576        &self,
12577        buffer: &ComputedWriteBuffer,
12578    ) -> ComputedWriteCoalescingPlan {
12579        self.plan_computed_write_coalescing_from_writes(
12580            buffer.writes().iter().cloned(),
12581            buffer.formats_present,
12582        )
12583    }
12584
12585    fn plan_owned_computed_write_coalescing(
12586        &self,
12587        writes: Vec<ComputedWrite>,
12588        formats_present: bool,
12589    ) -> ComputedWriteCoalescingPlan {
12590        self.plan_computed_write_coalescing_from_writes(writes, formats_present)
12591    }
12592
12593    fn plan_computed_write_coalescing_from_writes(
12594        &self,
12595        writes: impl IntoIterator<Item = ComputedWrite>,
12596        formats_present: bool,
12597    ) -> ComputedWriteCoalescingPlan {
12598        let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
12599            BTreeMap::new();
12600        let mut input_cells = 0usize;
12601        // One sheet lookup per sheet run of writes, not per cell.
12602        let mut located: Option<(SheetId, Option<&crate::arrow_store::ArrowSheet>)> = None;
12603
12604        for write in writes {
12605            match write {
12606                ComputedWrite::Cell {
12607                    seq,
12608                    sheet_id,
12609                    row0,
12610                    col0,
12611                    value,
12612                    format_id,
12613                } => {
12614                    input_cells = input_cells.saturating_add(1);
12615                    let sheet = match located {
12616                        Some((id, sheet)) if id == sheet_id => sheet,
12617                        _ => {
12618                            let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12619                            located = Some((sheet_id, sheet));
12620                            sheet
12621                        }
12622                    };
12623                    let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12624                        Some(sheet) => {
12625                            Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize)
12626                        }
12627                        None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12628                            row0 as usize,
12629                            32 * 1024,
12630                        ),
12631                    };
12632                    groups
12633                        .entry(ComputedWriteChunkKey {
12634                            sheet_id,
12635                            col0,
12636                            chunk_idx,
12637                            chunk_start_row0,
12638                        })
12639                        .or_default()
12640                        .push(ComputedWriteChunkEntryPlan {
12641                            row_in_chunk,
12642                            seq,
12643                            value,
12644                            format_id,
12645                        });
12646                }
12647                ComputedWrite::Run {
12648                    seq,
12649                    sheet_id,
12650                    row0,
12651                    col0,
12652                    entries,
12653                } => {
12654                    input_cells = input_cells.saturating_add(entries.len());
12655                    let sheet = match located {
12656                        Some((id, sheet)) if id == sheet_id => sheet,
12657                        _ => {
12658                            let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12659                            located = Some((sheet_id, sheet));
12660                            sheet
12661                        }
12662                    };
12663                    // Rows are located one by one (a binary search, no sheet
12664                    // lookup); the group map is touched once per chunk
12665                    // segment of the run.
12666                    let mut segment: Vec<ComputedWriteChunkEntryPlan> = Vec::new();
12667                    let mut segment_key: Option<ComputedWriteChunkKey> = None;
12668                    for (k, (value, format_id)) in entries.into_iter().enumerate() {
12669                        let row = row0.saturating_add(k as u32) as usize;
12670                        let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12671                            Some(sheet) => {
12672                                Self::locate_row_in_sheet_for_computed_write_plan(sheet, row)
12673                            }
12674                            None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12675                                row,
12676                                32 * 1024,
12677                            ),
12678                        };
12679                        let key = ComputedWriteChunkKey {
12680                            sheet_id,
12681                            col0,
12682                            chunk_idx,
12683                            chunk_start_row0,
12684                        };
12685                        if segment_key != Some(key)
12686                            && let Some(done) = segment_key.replace(key)
12687                        {
12688                            groups.entry(done).or_default().append(&mut segment);
12689                        }
12690                        segment.push(ComputedWriteChunkEntryPlan {
12691                            row_in_chunk,
12692                            seq,
12693                            value,
12694                            format_id,
12695                        });
12696                    }
12697                    if let Some(done) = segment_key {
12698                        groups.entry(done).or_default().append(&mut segment);
12699                    }
12700                }
12701                ComputedWrite::Rect {
12702                    seq,
12703                    sheet_id,
12704                    sr0,
12705                    sc0,
12706                    values,
12707                } => {
12708                    for (r_off, row) in values.into_iter().enumerate() {
12709                        for (c_off, value) in row.into_iter().enumerate() {
12710                            input_cells = input_cells.saturating_add(1);
12711                            self.push_computed_write_plan_entry(
12712                                &mut groups,
12713                                seq,
12714                                sheet_id,
12715                                sr0.saturating_add(r_off as u32),
12716                                sc0.saturating_add(c_off as u32),
12717                                value,
12718                                None,
12719                            );
12720                        }
12721                    }
12722                }
12723            }
12724        }
12725
12726        let mut plan = ComputedWriteCoalescingPlan {
12727            chunks: Vec::with_capacity(groups.len()),
12728            input_cells,
12729            coalesced_cells: 0,
12730            overwritten_cells: 0,
12731        };
12732        for (key, entries) in groups {
12733            let computed_lane_has_formats =
12734                self.computed_overlay_chunk_has_formats(key.sheet_id, key.col0, key.chunk_idx);
12735            let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(
12736                key,
12737                entries,
12738                formats_present,
12739                computed_lane_has_formats,
12740            );
12741            #[cfg(test)]
12742            if matches!(
12743                &chunk_plan.format_effect,
12744                ComputedWriteChunkFormatEffect::SetExplicit(_)
12745            ) {
12746                self.computed_format_vector_allocations_for_test
12747                    .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12748            }
12749            plan.coalesced_cells = plan
12750                .coalesced_cells
12751                .saturating_add(chunk_plan.entries.len());
12752            plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
12753            plan.chunks.push(chunk_plan);
12754        }
12755        debug_assert_eq!(
12756            plan.input_cells,
12757            plan.coalesced_cells.saturating_add(plan.overwritten_cells)
12758        );
12759        plan
12760    }
12761
12762    fn push_computed_write_plan_entry(
12763        &self,
12764        groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
12765        seq: u64,
12766        sheet_id: SheetId,
12767        row0: u32,
12768        col0: u32,
12769        value: OverlayValue,
12770        format_id: Option<crate::format::FormatId>,
12771    ) {
12772        let (chunk_idx, chunk_start_row0, row_in_chunk) =
12773            self.locate_computed_write_chunk(sheet_id, row0);
12774        let key = ComputedWriteChunkKey {
12775            sheet_id,
12776            col0,
12777            chunk_idx,
12778            chunk_start_row0,
12779        };
12780        groups
12781            .entry(key)
12782            .or_default()
12783            .push(ComputedWriteChunkEntryPlan {
12784                row_in_chunk,
12785                seq,
12786                value,
12787                format_id,
12788            });
12789    }
12790
12791    fn computed_overlay_chunk_has_formats(
12792        &self,
12793        sheet_id: SheetId,
12794        col0: u32,
12795        chunk_idx: usize,
12796    ) -> bool {
12797        let sheet_name = self.graph.sheet_name(sheet_id);
12798        self.arrow_sheets
12799            .sheet(sheet_name)
12800            .and_then(|sheet| sheet.columns.get(col0 as usize))
12801            .and_then(|column| column.chunk(chunk_idx))
12802            .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12803    }
12804
12805    fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
12806        let sheet_name = self.graph.sheet_name(sheet_id);
12807        if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
12808            return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
12809        }
12810        Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
12811    }
12812
12813    fn locate_row_in_sheet_for_computed_write_plan(
12814        sheet: &crate::arrow_store::ArrowSheet,
12815        row0: usize,
12816    ) -> (usize, u32, usize) {
12817        if row0 < sheet.nrows as usize
12818            && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
12819        {
12820            let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
12821            return (chunk_idx, chunk_start as u32, row_in_chunk);
12822        }
12823
12824        let chunk_rows = sheet.chunk_rows.max(1);
12825        if sheet.chunk_starts.is_empty() {
12826            return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
12827        }
12828
12829        let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
12830        let mut chunk_start = sheet.chunk_starts[chunk_idx];
12831        while chunk_start.saturating_add(chunk_rows) <= row0 {
12832            chunk_idx = chunk_idx.saturating_add(1);
12833            chunk_start = chunk_start.saturating_add(chunk_rows);
12834        }
12835        (
12836            chunk_idx,
12837            chunk_start as u32,
12838            row0.saturating_sub(chunk_start),
12839        )
12840    }
12841
12842    fn locate_row_in_empty_sheet_for_computed_write_plan(
12843        row0: usize,
12844        chunk_rows: usize,
12845    ) -> (usize, u32, usize) {
12846        let chunk_rows = chunk_rows.max(1);
12847        let chunk_idx = row0 / chunk_rows;
12848        let chunk_start = chunk_idx.saturating_mul(chunk_rows);
12849        (
12850            chunk_idx,
12851            chunk_start as u32,
12852            row0.saturating_sub(chunk_start),
12853        )
12854    }
12855
12856    #[cfg(test)]
12857    pub(crate) fn debug_plan_computed_write_coalescing(
12858        &self,
12859        buffer: &ComputedWriteBuffer,
12860    ) -> ComputedWriteCoalescingPlan {
12861        self.plan_computed_write_coalescing(buffer)
12862    }
12863
12864    pub(crate) fn flush_computed_write_buffer(
12865        &mut self,
12866        buffer: &mut ComputedWriteBuffer,
12867    ) -> Result<(), ExcelError> {
12868        if buffer.is_empty() {
12869            return Ok(());
12870        }
12871
12872        // Keep ownership of all pending writes until the final request
12873        // checkpoint and bounded commit-window preflight succeed so failures
12874        // remain retry safe. The synchronization and flush that follow are
12875        // infallible mutations with no cancellation point.
12876        self.resource_checkpoint(0)?;
12877        let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
12878        let (writes, formats_present) = buffer.take_writes();
12879        let plan = self.plan_owned_computed_write_coalescing(writes, formats_present);
12880        self.flush_computed_write_plan(plan);
12881        self.observe_evaluation_commit_window(commit_started);
12882
12883        Ok(())
12884    }
12885
12886    fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
12887        for chunk in plan.chunks {
12888            self.flush_computed_write_chunk_plan(chunk);
12889        }
12890    }
12891
12892    fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
12893        match &chunk.shape {
12894            ComputedWriteChunkPlanShape::Point => {
12895                self.flush_computed_write_chunk_plan_as_points(chunk);
12896            }
12897            ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
12898                self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
12899            }
12900            ComputedWriteChunkPlanShape::DenseRange { .. } => {
12901                self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12902            }
12903            ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
12904                if Self::should_emit_computed_run_fragment(*len, *runs) {
12905                    self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
12906                } else {
12907                    self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12908                }
12909            }
12910        }
12911    }
12912
12913    #[inline]
12914    fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
12915        runs <= len / 2
12916    }
12917
12918    fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
12919        let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
12920        for entry in chunk.entries {
12921            let row0 = chunk
12922                .chunk_start_row0
12923                .saturating_add(entry.row_in_chunk as u32);
12924            self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
12925        }
12926        self.apply_computed_overlay_format_effect(
12927            chunk.sheet_id,
12928            chunk.col0,
12929            chunk.chunk_idx,
12930            chunk.format_effect,
12931        );
12932    }
12933
12934    fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
12935        &mut self,
12936        chunk: ComputedWriteChunkPlan,
12937    ) {
12938        let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
12939        let sheet_id = chunk.sheet_id;
12940        let col0 = chunk.col0;
12941        let chunk_idx = chunk.chunk_idx;
12942        let chunk_start_row0 = chunk.chunk_start_row0;
12943        let format_effect = chunk.format_effect;
12944        let items: Vec<(usize, OverlayValue)> = chunk
12945            .entries
12946            .into_iter()
12947            .map(|entry| (entry.row_in_chunk, entry.value))
12948            .collect();
12949        match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
12950            items,
12951            point_estimate,
12952        ) {
12953            Some(Ok(fragment)) => {
12954                self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
12955            }
12956            Some(Err(cells)) => {
12957                self.flush_computed_overlay_cells_as_points(
12958                    sheet_id,
12959                    col0,
12960                    chunk_start_row0,
12961                    cells,
12962                );
12963            }
12964            None => {}
12965        }
12966        self.apply_computed_overlay_format_effect(sheet_id, col0, chunk_idx, format_effect);
12967    }
12968
12969    #[inline]
12970    fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
12971        chunk
12972            .entries
12973            .iter()
12974            .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
12975            .fold(0usize, usize::saturating_add)
12976    }
12977
12978    fn flush_computed_overlay_cells_as_points(
12979        &mut self,
12980        sheet_id: SheetId,
12981        col0: u32,
12982        chunk_start_row0: u32,
12983        cells: Vec<(usize, OverlayValue)>,
12984    ) {
12985        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
12986        for (row_in_chunk, value) in cells {
12987            let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
12988            self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
12989        }
12990    }
12991
12992    fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
12993        if chunk.entries.is_empty() {
12994            return;
12995        }
12996        let start = chunk.entries[0].row_in_chunk;
12997        let values: Vec<OverlayValue> =
12998            chunk.entries.into_iter().map(|entry| entry.value).collect();
12999        if let Some(fragment) = OverlayFragment::dense_range(start, values) {
13000            self.apply_computed_overlay_fragment(
13001                chunk.sheet_id,
13002                chunk.col0,
13003                chunk.chunk_idx,
13004                fragment,
13005            );
13006        }
13007        self.apply_computed_overlay_format_effect(
13008            chunk.sheet_id,
13009            chunk.col0,
13010            chunk.chunk_idx,
13011            chunk.format_effect,
13012        );
13013    }
13014
13015    fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
13016        if chunk.entries.is_empty() {
13017            return;
13018        }
13019        let start = chunk.entries[0].row_in_chunk;
13020        let values: Vec<OverlayValue> =
13021            chunk.entries.into_iter().map(|entry| entry.value).collect();
13022        if let Some(fragment) = OverlayFragment::run_range(start, values) {
13023            self.apply_computed_overlay_fragment(
13024                chunk.sheet_id,
13025                chunk.col0,
13026                chunk.chunk_idx,
13027                fragment,
13028            );
13029        }
13030        self.apply_computed_overlay_format_effect(
13031            chunk.sheet_id,
13032            chunk.col0,
13033            chunk.chunk_idx,
13034            chunk.format_effect,
13035        );
13036    }
13037
13038    fn apply_computed_overlay_format_effect(
13039        &mut self,
13040        sheet_id: SheetId,
13041        col0: u32,
13042        chunk_idx: usize,
13043        effect: ComputedWriteChunkFormatEffect,
13044    ) {
13045        if !(self.config.arrow_storage_enabled
13046            && self.config.delta_overlay_enabled
13047            && self.config.write_formula_overlay_enabled)
13048            || self.computed_overlay_mirroring_disabled
13049        {
13050            return;
13051        }
13052
13053        let sheet_name = self.graph.sheet_name(sheet_id);
13054        let Some(sheet) = self.arrow_sheets.sheet_mut(sheet_name) else {
13055            return;
13056        };
13057        let Some(chunk) = sheet
13058            .columns
13059            .get_mut(col0 as usize)
13060            .and_then(|column| column.chunk_mut(chunk_idx))
13061        else {
13062            return;
13063        };
13064        match effect {
13065            ComputedWriteChunkFormatEffect::NoFormatWork => {}
13066            ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Range {
13067                start,
13068                end,
13069            }) => {
13070                #[cfg(test)]
13071                {
13072                    self.computed_overlay_stale_clear_range_effects_for_test = self
13073                        .computed_overlay_stale_clear_range_effects_for_test
13074                        .saturating_add(1);
13075                }
13076                chunk.computed_overlay.clear_format_range(start, end);
13077            }
13078            ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Offsets(
13079                offsets,
13080            )) => {
13081                #[cfg(test)]
13082                {
13083                    self.computed_overlay_stale_clear_offset_attempts_for_test = self
13084                        .computed_overlay_stale_clear_offset_attempts_for_test
13085                        .saturating_add(offsets.len() as u64);
13086                }
13087                chunk.computed_overlay.clear_format_offsets(&offsets);
13088            }
13089            ComputedWriteChunkFormatEffect::SetExplicit(formats) => {
13090                #[cfg(test)]
13091                {
13092                    self.computed_overlay_set_explicit_entry_operations_for_test = self
13093                        .computed_overlay_set_explicit_entry_operations_for_test
13094                        .saturating_add(formats.len() as u64);
13095                }
13096                for (row_in_chunk, format_id) in formats {
13097                    chunk.computed_overlay.set_format(row_in_chunk, format_id);
13098                }
13099            }
13100        }
13101    }
13102
13103    fn apply_computed_overlay_fragment(
13104        &mut self,
13105        sheet_id: SheetId,
13106        col0: u32,
13107        chunk_idx: usize,
13108        fragment: OverlayFragment,
13109    ) {
13110        if !(self.config.arrow_storage_enabled
13111            && self.config.delta_overlay_enabled
13112            && self.config.write_formula_overlay_enabled)
13113        {
13114            return;
13115        }
13116        if self.computed_overlay_mirroring_disabled {
13117            return;
13118        }
13119
13120        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13121        self.ensure_arrow_sheet(&sheet_name);
13122
13123        let col0 = col0 as usize;
13124        let asheet = self
13125            .arrow_sheets
13126            .sheet_mut(&sheet_name)
13127            .expect("ArrowSheet must exist");
13128
13129        let cur_cols = asheet.columns.len();
13130        if col0 >= cur_cols {
13131            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
13132        }
13133
13134        let start_row0 = asheet
13135            .chunk_starts
13136            .get(chunk_idx)
13137            .copied()
13138            .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
13139        let required_rows =
13140            start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
13141        if required_rows > asheet.nrows as usize {
13142            if asheet.columns.is_empty() {
13143                asheet.insert_columns(0, 1);
13144            }
13145            asheet.ensure_row_capacity(required_rows);
13146        }
13147
13148        let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
13149            return;
13150        };
13151        let delta = ch.computed_overlay.apply_fragment(fragment);
13152        self.adjust_computed_overlay_bytes(delta);
13153
13154        if let Some(cap) = self.config.max_overlay_memory_bytes
13155            && self.computed_overlay_bytes_estimate > cap
13156        {
13157            self.disable_computed_overlay_mirroring_due_to_budget(cap);
13158        }
13159    }
13160
13161    #[inline]
13162    fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
13163        if delta >= 0 {
13164            self.computed_overlay_bytes_estimate = self
13165                .computed_overlay_bytes_estimate
13166                .saturating_add(delta as usize);
13167        } else {
13168            self.computed_overlay_bytes_estimate = self
13169                .computed_overlay_bytes_estimate
13170                .saturating_sub((-delta) as usize);
13171        }
13172    }
13173
13174    fn clear_all_computed_overlays(&mut self) {
13175        let mut freed_total = 0usize;
13176        for sh in self.arrow_sheets.sheets.iter_mut() {
13177            for col in sh.columns.iter_mut() {
13178                for ch in col.chunks.iter_mut() {
13179                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
13180                }
13181                for ch in col.sparse_chunks.values_mut() {
13182                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
13183                }
13184            }
13185        }
13186        self.computed_overlay_bytes_estimate = self
13187            .computed_overlay_bytes_estimate
13188            .saturating_sub(freed_total);
13189    }
13190
13191    fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
13192        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
13193        // Handle budget pressure by compacting computed overlays into base lanes.
13194        self.compact_all_computed_overlays();
13195    }
13196
13197    /// Fold all computed overlay entries across all sheets into their base arrays.
13198    /// This preserves data while freeing overlay memory, allowing mirroring to continue.
13199    fn compact_all_computed_overlays(&mut self) {
13200        let mut freed_total = 0usize;
13201        for sheet in self.arrow_sheets.sheets.iter_mut() {
13202            for col_idx in 0..sheet.columns.len() {
13203                // Dense chunks
13204                let num_dense = sheet.columns[col_idx].chunks.len();
13205                for ch_idx in 0..num_dense {
13206                    freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
13207                }
13208                // Sparse chunks
13209                let sparse_keys: Vec<usize> = sheet.columns[col_idx]
13210                    .sparse_chunks
13211                    .keys()
13212                    .copied()
13213                    .collect();
13214                for ch_idx in sparse_keys {
13215                    freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
13216                }
13217            }
13218        }
13219        self.computed_overlay_bytes_estimate = self
13220            .computed_overlay_bytes_estimate
13221            .saturating_sub(freed_total);
13222        self.overlay_compactions = self.overlay_compactions.saturating_add(1);
13223    }
13224
13225    fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
13226        let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
13227    }
13228
13229    fn record_vertex_value_to_overlay(
13230        &mut self,
13231        vertex_id: VertexId,
13232        value: &LiteralValue,
13233        computed_writes: Option<&mut ComputedWriteBuffer>,
13234    ) -> Result<(), ExcelError> {
13235        if !(self.config.arrow_storage_enabled
13236            && self.config.delta_overlay_enabled
13237            && self.config.write_formula_overlay_enabled)
13238        {
13239            return Ok(());
13240        }
13241        if self.computed_overlay_mirroring_disabled {
13242            return Ok(());
13243        }
13244        if !matches!(
13245            self.graph.get_vertex_kind(vertex_id),
13246            VertexKind::FormulaScalar | VertexKind::FormulaArray
13247        ) {
13248            return Ok(());
13249        }
13250        let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
13251            return Ok(());
13252        };
13253        let Some(buffer) = computed_writes else {
13254            // Unbuffered: one sheet lookup for the value and its format.
13255            let format_id = self.derived_formats.get(&cell);
13256            self.write_computed_cell_0based(cell, value, format_id);
13257            return Ok(());
13258        };
13259        let date_system = self
13260            .arrow_sheet_by_id(cell.sheet_id)
13261            .map(|sheet| sheet.date_system)
13262            .unwrap_or(self.config.date_system);
13263        let ov = Self::literal_to_overlay_value(value, date_system);
13264        {
13265            let format_id = self.derived_formats.get(&cell);
13266            buffer.push_cell_with_format(
13267                cell.sheet_id,
13268                cell.coord.row(),
13269                cell.coord.col(),
13270                ov,
13271                format_id,
13272            );
13273            if self.should_flush_computed_write_buffer(buffer) {
13274                self.flush_computed_write_buffer(buffer)?;
13275            }
13276        }
13277        Ok(())
13278    }
13279
13280    #[inline]
13281    fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
13282        self.config.max_overlay_memory_bytes.is_some_and(|cap| {
13283            if cap == 0 {
13284                return false;
13285            }
13286            self.computed_overlay_bytes_estimate
13287                .saturating_add(buffer.estimated_bytes())
13288                > cap
13289        })
13290    }
13291
13292    /// Estimated memory usage for computed overlays (formula/spill mirroring).
13293    pub fn overlay_memory_usage(&self) -> usize {
13294        self.computed_overlay_bytes_estimate
13295    }
13296
13297    #[cfg(test)]
13298    pub(crate) fn debug_overlay_compactions(&self) -> u64 {
13299        self.overlay_compactions
13300    }
13301
13302    #[cfg(test)]
13303    pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
13304        let mut total = 0usize;
13305        for sheet in &self.arrow_sheets.sheets {
13306            for column in &sheet.columns {
13307                for chunk in &column.chunks {
13308                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13309                }
13310                for chunk in column.sparse_chunks.values() {
13311                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13312                }
13313            }
13314        }
13315        self.computed_overlay_bytes_estimate = total;
13316        total
13317    }
13318
13319    fn resolve_sheet_locator_for_write(
13320        &mut self,
13321        loc: formualizer_common::SheetLocator<'_>,
13322        current_sheet: &str,
13323    ) -> Result<SheetId, ExcelError> {
13324        Ok(match loc {
13325            formualizer_common::SheetLocator::Id(id) => id,
13326            formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
13327            formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
13328        })
13329    }
13330
13331    fn resolve_sheet_locator_for_read(
13332        &self,
13333        loc: formualizer_common::SheetLocator<'_>,
13334        current_sheet: &str,
13335    ) -> Result<SheetId, ExcelError> {
13336        match loc {
13337            formualizer_common::SheetLocator::Id(id) => Ok(id),
13338            formualizer_common::SheetLocator::Name(name) => self
13339                .graph
13340                .sheet_id(name.as_ref())
13341                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13342            formualizer_common::SheetLocator::Current => self
13343                .graph
13344                .sheet_id(current_sheet)
13345                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13346        }
13347    }
13348
13349    /// Set a cell value
13350    pub fn set_cell_value(
13351        &mut self,
13352        sheet: &str,
13353        row: u32,
13354        col: u32,
13355        value: LiteralValue,
13356    ) -> Result<(), ExcelError> {
13357        self.observe_function_semantic_epoch()?;
13358        let sheet_existed = self.graph.sheet_id(sheet).is_some();
13359        let sheet_id = self.graph.sheet_id_mut(sheet);
13360        let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13361        let replaced = self
13362            .graph
13363            .get_vertex_id_for_address(&cell_ref)
13364            .filter(|&vertex| {
13365                matches!(
13366                    self.graph.get_vertex_kind(vertex),
13367                    VertexKind::FormulaScalar | VertexKind::FormulaArray
13368                )
13369            });
13370        let replaced_formula = replaced.is_some();
13371        // A value replacing a spilled anchor takes its spill with it (no
13372        // evaluation will revisit the anchor); anchors that spill blocked
13373        // wake through the cleared region.
13374        if let Some(anchor) = replaced
13375            && self.graph.spill_registry_has_anchor(anchor)
13376        {
13377            self.clear_spill_projection_and_mirror(anchor, None);
13378        }
13379        self.graph.set_cell_value(sheet, row, col, value.clone())?;
13380        self.clear_cell_format_state(sheet, cell_ref);
13381        self.record_changed_cell(sheet, row, col);
13382        if !sheet_existed || replaced_formula {
13383            self.mark_topology_edited();
13384        }
13385        // Mirror into Arrow overlay when enabled
13386        self.mirror_value_to_overlay(sheet, row, col, &value);
13387        // Advance snapshot to reflect external mutation.
13388        self.mark_data_edited();
13389        Ok(())
13390    }
13391
13392    /// Record a single-cell change: invalidates pending spills blocked on it.
13393    fn record_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
13394        let sheet_id = self.graph.sheet_id_mut(sheet);
13395        self.record_structural_change(StructuralScope::Cell {
13396            sheet: sheet_id,
13397            row: row.saturating_sub(1),
13398            col: col.saturating_sub(1),
13399        });
13400    }
13401
13402    fn record_change_for_event(&mut self, event: &ChangeEvent) {
13403        match event {
13404            ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
13405                self.record_structural_change(StructuralScope::Cell {
13406                    sheet: addr.sheet_id,
13407                    row: addr.coord.row(),
13408                    col: addr.coord.col(),
13409                });
13410            }
13411            ChangeEvent::SpillCommitted { new, .. } => {
13412                if let Some(scope) = Self::structural_scope_from_cells(&new.target_cells) {
13413                    self.record_structural_change(scope);
13414                }
13415            }
13416            ChangeEvent::SpillCleared { old, .. } => {
13417                if let Some(scope) = Self::structural_scope_from_cells(&old.target_cells) {
13418                    self.record_structural_change(scope);
13419                }
13420            }
13421            ChangeEvent::DefineName { .. }
13422            | ChangeEvent::UpdateName { .. }
13423            | ChangeEvent::DeleteName { .. }
13424            | ChangeEvent::NamedRangeAdjusted { .. } => {
13425                // Direct name events are preflighted by the logged-name APIs.
13426                // Structural entry points preflight spans before emitting a
13427                // NamedRangeAdjusted event. Epoch changes only rebuild caches.
13428                self.record_structural_change(StructuralScope::AllSheets);
13429            }
13430            ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
13431                // Structural entry points publish their axis delta once after
13432                // the graph and Arrow commits.
13433            }
13434            ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
13435                self.record_structural_change(StructuralScope::Region(Region::whole_row(
13436                    *sheet_id, *row0,
13437                )));
13438            }
13439            ChangeEvent::AddVertex { .. }
13440            | ChangeEvent::RemoveVertex { .. }
13441            | ChangeEvent::EdgeAdded { .. }
13442            | ChangeEvent::EdgeRemoved { .. }
13443            | ChangeEvent::CompoundStart { .. }
13444            | ChangeEvent::CompoundEnd { .. }
13445            | ChangeEvent::StagedFormulaCellChanged { .. } => {}
13446        }
13447    }
13448
13449    fn record_structural_change(&mut self, scope: StructuralScope) {
13450        if let StructuralScope::Cell { sheet, row, col } = scope {
13451            self.wake_spill_owner_of_entered_formula(sheet, row, col);
13452        }
13453        self.wake_spill_blocker_waiters(scope);
13454        self.invalidate_pending_spills(scope);
13455    }
13456
13457    /// Wake anchors waiting on a blocking cell inside `scope`, with their
13458    /// dependents: the anchor's value changes when it spills. A woken anchor
13459    /// registers again if it is still blocked.
13460    fn wake_spill_blocker_waiters(&mut self, scope: StructuralScope) {
13461        if self.spill_blocker_waiters.is_empty() {
13462            return;
13463        }
13464        let woken = self.spill_blocker_waiters.take_affected(scope);
13465        let woken: Vec<VertexId> = woken
13466            .into_iter()
13467            .filter(|&vertex| {
13468                self.graph.vertex_exists(vertex)
13469                    && matches!(
13470                        self.graph.get_vertex_kind(vertex),
13471                        VertexKind::FormulaScalar | VertexKind::FormulaArray
13472                    )
13473            })
13474            .collect();
13475        if !woken.is_empty() {
13476            self.graph.mark_dirty_many(&woken);
13477        }
13478    }
13479
13480    /// A formula entered into a live spill blocks it: wake the owning anchor
13481    /// so its next recalc re-plans (and reports `#SPILL!`). The spill keeps
13482    /// its registry until then; clearing it leaves the formula cell alone.
13483    fn wake_spill_owner_of_entered_formula(&mut self, sheet: SheetId, row: u32, col: u32) {
13484        let cell = self.graph.make_cell_ref_internal(sheet, row, col);
13485        if let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
13486            && self.graph.is_foreign_formula_cell(&cell, owner)
13487        {
13488            self.graph.mark_dirty_many(&[owner]);
13489        }
13490    }
13491
13492    fn structural_scope_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
13493        let first = cells.first()?;
13494        let sheet_id = first.sheet_id;
13495        if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
13496            return Some(StructuralScope::OpaqueGlobal);
13497        }
13498        let mut row_start = first.coord.row();
13499        let mut row_end = row_start;
13500        let mut col_start = first.coord.col();
13501        let mut col_end = col_start;
13502        for cell in cells.iter().skip(1) {
13503            row_start = row_start.min(cell.coord.row());
13504            row_end = row_end.max(cell.coord.row());
13505            col_start = col_start.min(cell.coord.col());
13506            col_end = col_end.max(cell.coord.col());
13507        }
13508        Some(StructuralScope::Region(Region::rect(
13509            sheet_id, row_start, row_end, col_start, col_end,
13510        )))
13511    }
13512
13513    pub fn set_cell_value_ref(
13514        &mut self,
13515        cell: formualizer_common::SheetCellRef<'_>,
13516        current_sheet: &str,
13517        value: LiteralValue,
13518    ) -> Result<(), ExcelError> {
13519        let owned = cell.into_owned();
13520        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13521        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13522        self.set_cell_value(
13523            &sheet_name,
13524            owned.coord.row() + 1,
13525            owned.coord.col() + 1,
13526            value,
13527        )
13528    }
13529
13530    pub fn set_cell_formula_ref(
13531        &mut self,
13532        cell: formualizer_common::SheetCellRef<'_>,
13533        current_sheet: &str,
13534        ast: ASTNode,
13535    ) -> Result<(), ExcelError> {
13536        let owned = cell.into_owned();
13537        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13538        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13539        self.set_cell_formula(
13540            &sheet_name,
13541            owned.coord.row() + 1,
13542            owned.coord.col() + 1,
13543            ast,
13544        )
13545    }
13546
13547    pub fn get_cell_value_ref(
13548        &self,
13549        cell: formualizer_common::SheetCellRef<'_>,
13550        current_sheet: &str,
13551    ) -> Result<Option<LiteralValue>, ExcelError> {
13552        let owned = cell.into_owned();
13553        let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
13554        let sheet_name = self.graph.sheet_name(sheet_id);
13555        Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
13556    }
13557
13558    pub fn resolve_range_view_sheet_ref<'c>(
13559        &'c self,
13560        r: &formualizer_common::SheetRef<'_>,
13561        current_sheet: &str,
13562    ) -> Result<RangeView<'c>, ExcelError> {
13563        use formualizer_common::SheetLocator;
13564
13565        let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
13566            match loc {
13567                SheetLocator::Current => Ok(None),
13568                SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
13569                SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
13570            }
13571        };
13572
13573        let rt = match r {
13574            formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
13575                sheet: sheet_to_opt_name(cell.sheet.clone())?,
13576                row: cell.coord.row() + 1,
13577                col: cell.coord.col() + 1,
13578                row_abs: cell.coord.row_abs(),
13579                col_abs: cell.coord.col_abs(),
13580            },
13581            formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
13582                sheet: sheet_to_opt_name(range.sheet.clone())?,
13583                start_row: range.start_row.map(|b| b.index + 1),
13584                start_col: range.start_col.map(|b| b.index + 1),
13585                end_row: range.end_row.map(|b| b.index + 1),
13586                end_col: range.end_col.map(|b| b.index + 1),
13587                start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
13588                start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
13589                end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
13590                end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
13591            },
13592        };
13593
13594        crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
13595    }
13596
13597    /// Set a cell formula
13598    pub fn set_cell_formula(
13599        &mut self,
13600        sheet: &str,
13601        row: u32,
13602        col: u32,
13603        ast: ASTNode,
13604    ) -> Result<(), ExcelError> {
13605        self.observe_function_semantic_epoch()?;
13606        let sheet_id = self.graph.sheet_id_mut(sheet);
13607        let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13608        let ingested = {
13609            let mut pipeline = self.ingest_pipeline();
13610            pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
13611        };
13612        self.graph.set_cell_formula_with_plan(
13613            sheet,
13614            row,
13615            col,
13616            ingested.ast_id,
13617            &ingested.dep_plan,
13618            ingested.dep_plan.volatile,
13619            ingested.dep_plan.dynamic,
13620        )?;
13621        self.clear_cell_format_state(sheet, placement);
13622        self.record_changed_cell(sheet, row, col);
13623
13624        // If the cell previously held a user value in the delta overlay, it must not continue
13625        // to mask the formula result under Arrow-canonical reads (overlay precedence is
13626        // delta -> computed -> base). Remove the overlay entry instead of writing `Empty`,
13627        // because an explicit `Empty` overlay would still take precedence over computed values.
13628        self.clear_delta_overlay_cell(sheet, row, col);
13629
13630        // Advance snapshot to reflect external mutation
13631        self.mark_topology_edited();
13632        Ok(())
13633    }
13634
13635    /// Bulk set many formulas on a sheet. Skips per-cell snapshot bumping and minimizes edge rebuilds.
13636    pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
13637    where
13638        I: IntoIterator<Item = (u32, u32, ASTNode)>,
13639    {
13640        let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
13641        let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
13642        let sheet_id = self.graph.sheet_id_mut(sheet);
13643        let ingested = {
13644            let mut pipeline = self.ingest_pipeline();
13645            let inputs = collected.into_iter().map(|(row, col, ast)| {
13646                let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13647                (FormulaAstInput::Tree(ast), placement, None)
13648            });
13649            pipeline.ingest_batch(inputs)?
13650        };
13651        let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
13652            .into_iter()
13653            .map(|formula| {
13654                (
13655                    formula.placement.coord.row() + 1,
13656                    formula.placement.coord.col() + 1,
13657                    formula.ast_id,
13658                    formula.dep_plan,
13659                )
13660            })
13661            .collect();
13662        let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
13663        for (row, col) in edited_cells {
13664            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13665            self.clear_cell_format_state(sheet, cell);
13666            self.record_changed_cell(sheet, row, col);
13667        }
13668        // Single topology bump after batch
13669        if n > 0 {
13670            self.mark_topology_edited();
13671        }
13672        Ok(n)
13673    }
13674
13675    /// Declare the formula cell at `sheet!(row, col)` (1-based) a dynamic
13676    /// array anchor whose spill identity comes from the source document, for
13677    /// example an XLSX `t="array"` formula bound to XLDAPR cell metadata.
13678    ///
13679    /// This changes only spill references to the anchor (`A1#`,
13680    /// `_xlfn.ANCHORARRAY(A1)`). A declared anchor with a committed
13681    /// multi-cell spill resolves to that spill, exactly as an undeclared
13682    /// one. Without a committed spill, a declared anchor whose current
13683    /// result is a scalar (not an error, so not a blocked `#SPILL!`
13684    /// anchor, and not empty) resolves to its own cell as a 1x1 reference,
13685    /// matching a spilled 1x1 dynamic array. Every other case keeps the
13686    /// `#REF!` of an undeclared anchor. Undeclared cells are unaffected, and
13687    /// fresh 1x1 results are not given spill identity by function name.
13688    ///
13689    /// The declaration belongs to the formula vertex. It is dropped when the
13690    /// cell's formula is replaced, the cell is cleared or overwritten with a
13691    /// value, the vertex is removed (including with its sheet), or a
13692    /// structural row/column edit moves the vertex. A structural edit that
13693    /// only adjusts the formula's references, without moving the anchor,
13694    /// keeps it. Declaring again re-establishes it.
13695    ///
13696    /// The anchor and its dependents are marked dirty, so the next
13697    /// evaluation observes the declaration.
13698    ///
13699    /// Returns `#REF!` if the sheet does not exist or the cell holds no
13700    /// formula vertex. A formula still staged by deferred graph building has
13701    /// no vertex: build the graph (for example [`Self::build_graph_all`])
13702    /// before declaring.
13703    pub fn declare_dynamic_array_anchor(
13704        &mut self,
13705        sheet: &str,
13706        row: u32,
13707        col: u32,
13708    ) -> Result<(), ExcelError> {
13709        let not_a_formula = || {
13710            ExcelError::new(ExcelErrorKind::Ref)
13711                .with_message("Declared dynamic array anchor must be a formula cell")
13712        };
13713        if row == 0 || col == 0 {
13714            return Err(not_a_formula());
13715        }
13716        if self.get_staged_formula_text(sheet, row, col).is_some() {
13717            return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
13718                "Declared dynamic array anchor is still staged; build the graph first",
13719            ));
13720        }
13721        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(not_a_formula)?;
13722        let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13723        let vertex = self
13724            .graph
13725            .cell_vertex_mut(&cell)
13726            .filter(|v| self.graph.vertex_has_formula(*v))
13727            .ok_or_else(not_a_formula)?;
13728        if self.graph.declare_dynamic_anchor(vertex, cell) {
13729            self.graph.mark_dirty_many(&[vertex]);
13730            self.mark_topology_edited();
13731        }
13732        Ok(())
13733    }
13734
13735    /// Declare a legacy array formula with a fixed output rectangle after
13736    /// ingestion and before evaluation. Coordinates are one-based. Replacement,
13737    /// removal or movement of the formula clears the declaration.
13738    /// Single-cell declarations do not register a spill anchor. Requires
13739    /// `EvalConfig::family_execution = false` to bypass family memoization.
13740    pub fn declare_fixed_array_formula(
13741        &mut self,
13742        sheet: &str,
13743        row: u32,
13744        col: u32,
13745        rows: u32,
13746        cols: u32,
13747    ) -> Result<(), ExcelError> {
13748        if self.config.family_execution {
13749            return Err(ExcelError::new(ExcelErrorKind::Ref)
13750                .with_message("Fixed arrays require family_execution = false"));
13751        }
13752        if row == 0
13753            || col == 0
13754            || rows == 0
13755            || cols == 0
13756            || row.checked_add(rows - 1).is_none_or(|r| r > 1_048_576)
13757            || col.checked_add(cols - 1).is_none_or(|c| c > 16_384)
13758            || u64::from(rows) * u64::from(cols) > u64::from(self.config.spill.max_spill_cells)
13759        {
13760            return Err(ExcelError::new(ExcelErrorKind::Ref)
13761                .with_message("Invalid or over-cap fixed array extent"));
13762        }
13763        if self.get_staged_formula_text(sheet, row, col).is_some() {
13764            return Err(ExcelError::new(ExcelErrorKind::Ref)
13765                .with_message("Fixed array formula is still staged; build the graph first"));
13766        }
13767        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
13768            ExcelError::new(ExcelErrorKind::Ref).with_message("Unknown fixed array sheet")
13769        })?;
13770        let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13771        let vertex = self
13772            .graph
13773            .cell_vertex_mut(&cell)
13774            .filter(|v| self.graph.vertex_has_formula(*v))
13775            .ok_or_else(|| {
13776                ExcelError::new(ExcelErrorKind::Ref)
13777                    .with_message("Fixed array anchor must be a formula cell")
13778            })?;
13779        if (rows == 1 && cols == 1 && self.graph.fixed_single_arrays.contains(&vertex))
13780            || self.graph.fixed_array_shapes.get(&vertex) == Some(&(rows, cols))
13781        {
13782            return Ok(());
13783        }
13784        if self.graph.fixed_single_arrays.contains(&vertex)
13785            || self.graph.spill_registry_has_anchor(vertex)
13786            || self.graph.is_current_declared_dynamic_anchor(vertex)
13787        {
13788            return Err(ExcelError::new(ExcelErrorKind::Ref)
13789                .with_message("Array declaration already exists; replace the formula first"));
13790        }
13791        if rows == 1 && cols == 1 {
13792            self.graph.fixed_single_arrays.insert(vertex);
13793        } else {
13794            let mut targets = Vec::with_capacity((rows * cols) as usize);
13795            for r in 0..rows {
13796                for c in 0..cols {
13797                    targets.push(self.graph.make_cell_ref(sheet, row + r, col + c));
13798                }
13799            }
13800            self.graph.plan_spill_region(vertex, &targets)?;
13801            // Reserve ownership before evaluation, even for Empty members.
13802            self.graph
13803                .commit_spill_region_atomic_with_fault(vertex, targets, vec![], None)?;
13804            self.graph.fixed_array_shapes.insert(vertex, (rows, cols));
13805        }
13806        self.graph.mark_dirty_many(&[vertex]);
13807        self.mark_topology_edited();
13808        Ok(())
13809    }
13810
13811    #[inline]
13812    fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
13813        match v {
13814            LiteralValue::Empty => None,
13815            LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
13816            other => Some(other),
13817        }
13818    }
13819
13820    fn materialize_temporal_egress(
13821        value: LiteralValue,
13822        class: Option<&formualizer_common::numfmt::FormatClass>,
13823        policy: crate::engine::TemporalEgress,
13824        date_system: crate::engine::DateSystem,
13825    ) -> LiteralValue {
13826        use formualizer_common::numfmt::FormatClass;
13827        if policy == crate::engine::TemporalEgress::Serial {
13828            return value;
13829        }
13830        let LiteralValue::Number(serial) = value else {
13831            return value;
13832        };
13833        match class {
13834            Some(FormatClass::Date) if serial.fract() != 0.0 => {
13835                formualizer_common::try_serial_to_datetime_for(date_system, serial)
13836                    .map(LiteralValue::DateTime)
13837                    .unwrap_or(LiteralValue::Number(serial))
13838            }
13839            Some(FormatClass::Date) => {
13840                formualizer_common::try_serial_to_date_for(date_system, serial)
13841                    .map(LiteralValue::Date)
13842                    .unwrap_or(LiteralValue::Number(serial))
13843            }
13844            Some(FormatClass::DateTime) => {
13845                formualizer_common::try_serial_to_datetime_for(date_system, serial)
13846                    .map(LiteralValue::DateTime)
13847                    .unwrap_or(LiteralValue::Number(serial))
13848            }
13849            Some(FormatClass::Time) if (0.0..1.0).contains(&serial) => {
13850                let nanos = (serial * 86_400_000_000_000.0).round() as u64;
13851                let nanos = nanos.min(86_400_000_000_000 - 1);
13852                chrono::NaiveTime::from_num_seconds_from_midnight_opt(
13853                    (nanos / 1_000_000_000) as u32,
13854                    (nanos % 1_000_000_000) as u32,
13855                )
13856                .map(LiteralValue::Time)
13857                .unwrap_or(LiteralValue::Number(serial))
13858            }
13859            Some(FormatClass::Time | FormatClass::Duration) => {
13860                let nanos = (serial * 86_400.0 * 1_000_000_000.0).round();
13861                if nanos.is_finite() && nanos >= i64::MIN as f64 && nanos <= i64::MAX as f64 {
13862                    LiteralValue::Duration(chrono::Duration::nanoseconds(nanos as i64))
13863                } else {
13864                    LiteralValue::Number(serial)
13865                }
13866            }
13867            _ => LiteralValue::Number(serial),
13868        }
13869    }
13870
13871    pub(crate) fn effective_format_id(
13872        &self,
13873        sheet: &str,
13874        row: u32,
13875        col: u32,
13876    ) -> Option<crate::format::FormatId> {
13877        let arrow = self.arrow_sheets.sheet(sheet).and_then(|arrow| {
13878            arrow.format_id(
13879                row.saturating_sub(1) as usize,
13880                col.saturating_sub(1) as usize,
13881            )
13882        });
13883        arrow.or_else(|| {
13884            let sheet_id = self.graph.sheet_id(sheet)?;
13885            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13886            self.derived_formats.get(&cell)
13887        })
13888    }
13889
13890    /// Get a cell value through the single temporal egress boundary.
13891    pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13892        let raw = self.read_cell_value(sheet, row, col)?;
13893        let format = self.effective_format_id(sheet, row, col);
13894        let class = format.and_then(|id| self.format_registry.class(id));
13895        Self::normalize_public_cell_read(Self::materialize_temporal_egress(
13896            raw,
13897            class,
13898            self.config.temporal_egress,
13899            self.config.date_system,
13900        ))
13901    }
13902
13903    /// Read a rectangular range through the temporal egress boundary.
13904    pub fn get_range_values(
13905        &self,
13906        sheet: &str,
13907        sr: u32,
13908        sc: u32,
13909        er: u32,
13910        ec: u32,
13911    ) -> Vec<Vec<LiteralValue>> {
13912        let height = er.saturating_sub(sr).saturating_add(1) as usize;
13913        let width = ec.saturating_sub(sc).saturating_add(1) as usize;
13914        let Some(asheet) = self.sheet_store().sheet(sheet) else {
13915            return vec![vec![LiteralValue::Empty; width]; height];
13916        };
13917        let view = asheet.range_view(
13918            sr.saturating_sub(1) as usize,
13919            sc.saturating_sub(1) as usize,
13920            er.saturating_sub(1) as usize,
13921            ec.saturating_sub(1) as usize,
13922        );
13923        let sheet_id = self.graph.sheet_id(sheet);
13924        let derived_formats = &self.derived_formats;
13925        let has_derived_formats =
13926            sheet_id.is_some_and(|sheet_id| derived_formats.any(|cell| cell.sheet_id == sheet_id));
13927        let mut out = Vec::with_capacity(height);
13928        if !asheet.has_formats() && !has_derived_formats {
13929            for rr in 0..height {
13930                let mut row = Vec::with_capacity(width);
13931                for cc in 0..width {
13932                    row.push(view.get_cell(rr, cc));
13933                }
13934                out.push(row);
13935            }
13936            return out;
13937        }
13938        let format_registry = &self.format_registry;
13939        for rr in 0..height {
13940            let mut row = Vec::with_capacity(width);
13941            for cc in 0..width {
13942                let raw = view.get_cell(rr, cc);
13943                let row0 = sr.saturating_sub(1).saturating_add(rr as u32);
13944                let col0 = sc.saturating_sub(1).saturating_add(cc as u32);
13945                let format = asheet.format_id(row0 as usize, col0 as usize).or_else(|| {
13946                    let cell = CellRef::new(sheet_id?, Coord::new(row0, col0, true, true));
13947                    derived_formats.get(&cell)
13948                });
13949                let class = format.and_then(|id| format_registry.class(id));
13950                row.push(Self::materialize_temporal_egress(
13951                    raw,
13952                    class,
13953                    self.config.temporal_egress,
13954                    self.config.date_system,
13955                ));
13956            }
13957            out.push(row);
13958        }
13959        out
13960    }
13961
13962    /// Unified internal read API for a single cell value (Arrow-truth).
13963    /// The 1x1 spill fallback of a declared dynamic-array anchor: its
13964    /// committed result is a non-error, non-empty scalar. Scheduling orders
13965    /// the anchor before its spill readers (the reference keeps a static
13966    /// edge to it), exactly as for committed multi-cell spills.
13967    fn declared_anchor_holds_scalar(&self, anchor: CellRef) -> bool {
13968        let sheet = self.graph.sheet_name(anchor.sheet_id);
13969        if sheet.is_empty() {
13970            return false;
13971        }
13972        matches!(
13973            self.read_cell_value(sheet, anchor.coord.row() + 1, anchor.coord.col() + 1),
13974            Some(value) if !matches!(
13975                value,
13976                LiteralValue::Error(_)
13977                    | LiteralValue::Array(_)
13978                    | LiteralValue::Pending
13979                    | LiteralValue::Empty
13980            )
13981        )
13982    }
13983
13984    pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13985        let asheet = self.sheet_store().sheet(sheet)?;
13986        let r0 = row.saturating_sub(1) as usize;
13987        let c0 = col.saturating_sub(1) as usize;
13988        let v = asheet.get_cell_value(r0, c0);
13989        if matches!(v, LiteralValue::Empty) {
13990            None
13991        } else {
13992            Some(v)
13993        }
13994    }
13995
13996    /// Unified internal read API for a range of cell values (Arrow-truth).
13997    pub(crate) fn read_range_values(
13998        &self,
13999        sheet: &str,
14000        sr: u32,
14001        sc: u32,
14002        er: u32,
14003        ec: u32,
14004    ) -> RangeView<'_> {
14005        let Some(asheet) = self.sheet_store().sheet(sheet) else {
14006            return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
14007        };
14008        if er < sr || ec < sc {
14009            return asheet.range_view(1, 1, 0, 0);
14010        }
14011        let sr0 = sr.saturating_sub(1) as usize;
14012        let sc0 = sc.saturating_sub(1) as usize;
14013        let er0 = er.saturating_sub(1) as usize;
14014        let ec0 = ec.saturating_sub(1) as usize;
14015        asheet.range_view(sr0, sc0, er0, ec0)
14016    }
14017
14018    /// Get formula AST (if any) and current stored value for a cell
14019    pub fn get_cell(
14020        &self,
14021        sheet: &str,
14022        row: u32,
14023        col: u32,
14024    ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
14025        let v = self.get_cell_value(sheet, row, col);
14026        let sheet_id = self.graph.sheet_id(sheet)?;
14027        let coord = Coord::from_excel(row, col, true, true);
14028        let cell = CellRef::new(sheet_id, coord);
14029        if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
14030            let ast = self.graph.get_formula(vid);
14031            Some((ast, v))
14032        } else if v.is_some() || self.graph.had_legacy_cell_vertex(&cell) {
14033            // A referenced or emptied value cell has no vertex (decision
14034            // 27), but it is a cell the graph knows, as it was when it had
14035            // one (the interactive formula edit routes on this).
14036            Some((None, v))
14037        } else {
14038            None
14039        }
14040    }
14041
14042    /// Begin batch operations - defer CSR rebuilds for better performance
14043    pub fn begin_batch(&mut self) {
14044        self.graph.begin_batch();
14045    }
14046
14047    /// End batch operations and trigger CSR rebuild
14048    pub fn end_batch(&mut self) {
14049        self.graph.end_batch();
14050    }
14051
14052    /// Begin a deferred-dirty scope for a multi-edit batch: while active,
14053    /// every edit's dirty propagation queues its sources instead of running
14054    /// a full BFS per edit, and the outermost `end_deferred_dirty` flushes
14055    /// the union with ONE multi-source propagation (O(component) instead of
14056    /// O(edits × component)). See `DependencyGraph::begin_deferred_dirty`.
14057    ///
14058    /// Callers MUST run `end_deferred_dirty` on every exit path, including
14059    /// error returns; evaluation entry points `debug_assert` no scope leaked.
14060    pub fn begin_deferred_dirty(&mut self) {
14061        self.graph.begin_deferred_dirty();
14062    }
14063
14064    /// End a deferred-dirty scope, flushing the queued propagation when the
14065    /// outermost scope closes. See `Engine::begin_deferred_dirty`.
14066    pub fn end_deferred_dirty(&mut self) {
14067        let _ = self.graph.end_deferred_dirty();
14068    }
14069
14070    /// Total vertices processed by dirty-propagation BFS loops since graph
14071    /// creation. Perf-shape observability only (cross-crate tests assert
14072    /// batched edits propagate O(component), not O(edits × component)).
14073    pub fn dirty_propagation_visits(&self) -> u64 {
14074        self.graph.dirty_propagation_visits()
14075    }
14076
14077    /// Evaluate a single vertex.
14078    /// This is the core of the sequential evaluation logic for Milestone 3.1.
14079    #[inline]
14080    fn record_cell_if_changed(
14081        delta: &mut DeltaCollector,
14082        cell: &CellRef,
14083        old: &LiteralValue,
14084        new: &LiteralValue,
14085    ) {
14086        if old != new {
14087            delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
14088        }
14089    }
14090
14091    pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
14092        self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
14093            engine.observe_function_semantic_epoch()?;
14094            // A direct request selects exactly one vertex, regardless of its formula kind.
14095            engine.resource_checkpoint(1)?;
14096            if !engine.graph.vertex_exists(vertex_id) {
14097                return engine.evaluate_vertex_impl(vertex_id, None);
14098            }
14099            let is_formula = matches!(
14100                engine.graph.get_vertex_kind(vertex_id),
14101                VertexKind::FormulaScalar | VertexKind::FormulaArray
14102            );
14103            if is_formula {
14104                engine.begin_evaluation_request();
14105                #[cfg(any(test, feature = "legacy_oracle"))]
14106                engine.graph.flush_pending_edge_deltas();
14107                let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
14108                    vertex_id,
14109                )];
14110                engine.evaluate_legacy_target_roots(&roots, None)?;
14111            }
14112            if !is_formula {
14113                engine.refresh_resolution_snapshots();
14114            }
14115            engine.evaluate_vertex_impl(vertex_id, None)
14116        })
14117    }
14118
14119    /// Same rejection publication for owned arrays and pre-admitted range views.
14120    fn publish_oversized_spill(
14121        &mut self,
14122        vertex_id: VertexId,
14123        error: ExcelError,
14124        mut delta: Option<&mut DeltaCollector>,
14125    ) -> LiteralValue {
14126        self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14127        let anchor = self
14128            .graph
14129            .get_cell_ref(vertex_id)
14130            .expect("cell ref for vertex");
14131        let spill_val = LiteralValue::Error(error);
14132        if let Some(d) = delta {
14133            let old = self
14134                .read_cell_value(
14135                    self.graph.sheet_name(anchor.sheet_id),
14136                    anchor.coord.row() + 1,
14137                    anchor.coord.col() + 1,
14138                )
14139                .unwrap_or(LiteralValue::Empty);
14140            if old != spill_val {
14141                d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14142            }
14143        }
14144        self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14145        if self.config.arrow_storage_enabled
14146            && self.config.delta_overlay_enabled
14147            && self.config.write_formula_overlay_enabled
14148        {
14149            let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14150            self.mirror_value_to_computed_overlay(
14151                &sheet_name,
14152                anchor.coord.row() + 1,
14153                anchor.coord.col() + 1,
14154                &spill_val,
14155            );
14156        }
14157        spill_val
14158    }
14159
14160    fn evaluate_vertex_impl(
14161        &mut self,
14162        vertex_id: VertexId,
14163        delta: Option<&mut DeltaCollector>,
14164    ) -> Result<LiteralValue, ExcelError> {
14165        // Preserve the direct evaluator's compatibility behavior for invalid IDs, literal cells,
14166        // names, and other non-formula vertices. Only formula publication needs the C1a final
14167        // deadline checkpoint and effects pipeline.
14168        if !self.graph.vertex_exists(vertex_id) {
14169            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
14170                .with_message(format!("Vertex not found: {vertex_id:?}")));
14171        }
14172        if (self.active_resource_ledger.is_some()
14173            || self.graph.fixed_array_shapes.contains_key(&vertex_id))
14174            && matches!(
14175                self.graph.get_vertex_kind(vertex_id),
14176                VertexKind::FormulaScalar | VertexKind::FormulaArray
14177            )
14178        {
14179            let value = self
14180                .evaluate_vertex_immutable(vertex_id)
14181                .unwrap_or_else(LiteralValue::Error);
14182            let effects = self.plan_vertex_effects(vertex_id, value.clone())?;
14183            // Do not publish the selected result until the outer request's deadline succeeds.
14184            self.resource_checkpoint(0)?;
14185            let mut delta = delta;
14186            for effect in &effects {
14187                self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
14188            }
14189            return Ok(value);
14190        }
14191
14192        let mut delta = delta;
14193
14194        // Get vertex kind and check if it needs evaluation
14195        let kind = self.graph.get_vertex_kind(vertex_id);
14196        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
14197
14198        let view = match kind {
14199            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
14200                if let Some(view) = self.graph.formula_view(vertex_id) {
14201                    view
14202                } else {
14203                    return Ok(LiteralValue::Number(0.0));
14204                }
14205            }
14206            VertexKind::Empty | VertexKind::Cell => {
14207                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
14208                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14209                    let row = cell_ref.coord.row() + 1;
14210                    let col = cell_ref.coord.col() + 1;
14211                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
14212                        return Ok(v);
14213                    }
14214                }
14215                return Ok(LiteralValue::Number(0.0));
14216            }
14217            VertexKind::NamedScalar => {
14218                let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
14219                return Ok(value);
14220            }
14221            VertexKind::NamedArray => {
14222                let value = self.evaluate_named_array(vertex_id, sheet_id)?;
14223                return Ok(value);
14224            }
14225            VertexKind::InfiniteRange
14226            | VertexKind::Range
14227            | VertexKind::External
14228            | VertexKind::Table => {
14229                // Not directly evaluatable here.
14230                return Ok(LiteralValue::Number(0.0));
14231            }
14232        };
14233
14234        // The interpreter uses a reference to the engine as the context.
14235        let sheet_name = self.graph.sheet_name(sheet_id);
14236        let cell_ref = self
14237            .graph
14238            .get_cell_ref(vertex_id)
14239            .expect("cell ref for vertex");
14240        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14241            .with_resolved_functions(&self.resolved_functions)
14242            .with_arena_cells(self);
14243
14244        let result = interpreter.evaluate_formula_view(
14245            view,
14246            self.graph.data_store(),
14247            self.graph.sheet_reg(),
14248        );
14249
14250        // If array result, perform spill from the anchor cell
14251        match result {
14252            Ok(cv) => {
14253                let derived_format = cv.format_id();
14254                self.record_derived_format(vertex_id, derived_format);
14255                let oversized_range = if matches!(cv, crate::traits::CalcValue::Range(_))
14256                    && self.graph.fixed_single_arrays.contains(&vertex_id)
14257                {
14258                    None
14259                } else {
14260                    crate::engine::result_finalization::range_spill_error(
14261                        &cv,
14262                        self.config.spill.max_spill_cells,
14263                    )
14264                };
14265                let is_oversized_range = oversized_range.is_some();
14266                let result_literal = if let Some(error) = oversized_range {
14267                    drop(cv);
14268                    LiteralValue::Error(error)
14269                } else {
14270                    self.materialize_formula_result(vertex_id, cv)
14271                };
14272                let output_sheet_name = sheet_name.to_string();
14273                self.write_computed_overlay_format_0based(
14274                    &output_sheet_name,
14275                    cell_ref.coord.row(),
14276                    cell_ref.coord.col(),
14277                    derived_format,
14278                );
14279                if is_oversized_range {
14280                    let LiteralValue::Error(error) = result_literal else {
14281                        unreachable!()
14282                    };
14283                    self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
14284                    return Ok(self.publish_oversized_spill(
14285                        vertex_id,
14286                        error,
14287                        delta.as_deref_mut(),
14288                    ));
14289                }
14290                match result_literal {
14291                    LiteralValue::Array(rows) => {
14292                        // Update kind to FormulaArray for tracking
14293                        self.graph
14294                            .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
14295                        // Build target cells rectangle starting from anchor
14296                        let anchor = self
14297                            .graph
14298                            .get_cell_ref(vertex_id)
14299                            .expect("cell ref for vertex");
14300                        let sheet_id = anchor.sheet_id;
14301                        let h = rows.len() as u32;
14302                        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
14303
14304                        // Hard cap to avoid vertex explosion from huge dynamic arrays.
14305                        let spill_cells = (h as u64).saturating_mul(w as u64);
14306                        if spill_cells > self.config.spill.max_spill_cells as u64 {
14307                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14308                                .with_message("SpillTooLarge")
14309                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14310                                    expected_rows: h,
14311                                    expected_cols: w,
14312                                });
14313                            return Ok(self.publish_oversized_spill(
14314                                vertex_id,
14315                                spill_err,
14316                                delta.as_deref_mut(),
14317                            ));
14318                        }
14319                        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity)
14320                        const PACKED_MAX_ROW: u32 = 1_048_575; // 20-bit max
14321                        const PACKED_MAX_COL: u32 = 16_383; // 14-bit max
14322                        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
14323                        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
14324                        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
14325                            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14326                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14327                                .with_message("Spill exceeds sheet bounds")
14328                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14329                                    expected_rows: h,
14330                                    expected_cols: w,
14331                                });
14332                            let spill_val = LiteralValue::Error(spill_err.clone());
14333                            if let Some(d) = delta.as_deref_mut() {
14334                                let old = self
14335                                    .read_cell_value(
14336                                        self.graph.sheet_name(anchor.sheet_id),
14337                                        anchor.coord.row() + 1,
14338                                        anchor.coord.col() + 1,
14339                                    )
14340                                    .unwrap_or(LiteralValue::Empty);
14341                                if old != spill_val {
14342                                    d.record_cell(
14343                                        anchor.sheet_id,
14344                                        anchor.coord.row(),
14345                                        anchor.coord.col(),
14346                                    );
14347                                }
14348                            }
14349                            self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14350                            if self.config.arrow_storage_enabled
14351                                && self.config.delta_overlay_enabled
14352                                && self.config.write_formula_overlay_enabled
14353                            {
14354                                let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14355                                self.mirror_value_to_computed_overlay(
14356                                    &sheet_name,
14357                                    anchor.coord.row() + 1,
14358                                    anchor.coord.col() + 1,
14359                                    &spill_val,
14360                                );
14361                            }
14362                            return Ok(spill_val);
14363                        }
14364                        let mut targets = Vec::new();
14365                        for r in 0..h {
14366                            for c in 0..w {
14367                                targets.push(self.graph.make_cell_ref_internal(
14368                                    sheet_id,
14369                                    anchor.coord.row() + r,
14370                                    anchor.coord.col() + c,
14371                                ));
14372                            }
14373                        }
14374
14375                        // Plan spill via spill manager shim
14376                        match self.spill_mgr.reserve(
14377                            vertex_id,
14378                            anchor,
14379                            SpillShape { rows: h, cols: w },
14380                            SpillMeta {
14381                                epoch: self.recalc_epoch,
14382                                config: self.config.spill,
14383                            },
14384                        ) {
14385                            Ok(()) => {
14386                                // Commit: write values to grid
14387                                // Default conflict policy is Error + FirstWins; reserve() enforces in-flight locks
14388                                // and plan_spill_region enforces overlap with committed formulas/spills/values.
14389                                if let Err(e) = self.commit_spill_and_mirror(
14390                                    vertex_id,
14391                                    &targets,
14392                                    rows.clone(),
14393                                    delta.as_deref_mut(),
14394                                ) {
14395                                    if e.kind != ExcelErrorKind::Spill {
14396                                        return Err(e);
14397                                    }
14398                                    // If commit fails, mark as error
14399                                    self.clear_spill_projection_and_mirror(
14400                                        vertex_id,
14401                                        delta.as_deref_mut(),
14402                                    );
14403                                    if let Some(d) = delta.as_deref_mut() {
14404                                        let old = self
14405                                            .read_cell_value(
14406                                                self.graph.sheet_name(anchor.sheet_id),
14407                                                anchor.coord.row() + 1,
14408                                                anchor.coord.col() + 1,
14409                                            )
14410                                            .unwrap_or(LiteralValue::Empty);
14411                                        let new = LiteralValue::Error(e.clone());
14412                                        if old != new {
14413                                            d.record_cell(
14414                                                anchor.sheet_id,
14415                                                anchor.coord.row(),
14416                                                anchor.coord.col(),
14417                                            );
14418                                        }
14419                                    }
14420                                    let err_val = LiteralValue::Error(e.clone());
14421                                    self.graph.update_vertex_value_ref(vertex_id, &err_val);
14422                                    if self.config.arrow_storage_enabled
14423                                        && self.config.delta_overlay_enabled
14424                                        && self.config.write_formula_overlay_enabled
14425                                    {
14426                                        let sheet_name =
14427                                            self.graph.sheet_name(anchor.sheet_id).to_string();
14428                                        self.mirror_value_to_computed_overlay(
14429                                            &sheet_name,
14430                                            anchor.coord.row() + 1,
14431                                            anchor.coord.col() + 1,
14432                                            &err_val,
14433                                        );
14434                                    }
14435                                    return Ok(err_val);
14436                                }
14437                                // Anchor shows the top-left value, like Excel
14438                                let top_left = rows
14439                                    .first()
14440                                    .and_then(|r| r.first())
14441                                    .cloned()
14442                                    .unwrap_or(LiteralValue::Empty);
14443                                self.graph.update_vertex_value_ref(vertex_id, &top_left);
14444                                Ok(top_left)
14445                            }
14446                            Err(e) => {
14447                                self.clear_spill_projection_and_mirror(
14448                                    vertex_id,
14449                                    delta.as_deref_mut(),
14450                                );
14451                                let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14452                                    .with_message(e.message().unwrap_or("Spill blocked"))
14453                                    .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14454                                        expected_rows: h,
14455                                        expected_cols: w,
14456                                    });
14457                                let spill_val = LiteralValue::Error(spill_err.clone());
14458                                if let Some(d) = delta.as_deref_mut() {
14459                                    let old = self
14460                                        .read_cell_value(
14461                                            self.graph.sheet_name(anchor.sheet_id),
14462                                            anchor.coord.row() + 1,
14463                                            anchor.coord.col() + 1,
14464                                        )
14465                                        .unwrap_or(LiteralValue::Empty);
14466                                    if old != spill_val {
14467                                        d.record_cell(
14468                                            anchor.sheet_id,
14469                                            anchor.coord.row(),
14470                                            anchor.coord.col(),
14471                                        );
14472                                    }
14473                                }
14474                                self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14475                                if self.config.arrow_storage_enabled
14476                                    && self.config.delta_overlay_enabled
14477                                    && self.config.write_formula_overlay_enabled
14478                                {
14479                                    let sheet_name =
14480                                        self.graph.sheet_name(anchor.sheet_id).to_string();
14481                                    self.mirror_value_to_computed_overlay(
14482                                        &sheet_name,
14483                                        anchor.coord.row() + 1,
14484                                        anchor.coord.col() + 1,
14485                                        &spill_val,
14486                                    );
14487                                }
14488                                Ok(spill_val)
14489                            }
14490                        }
14491                    }
14492                    other => {
14493                        // Scalar result: store value and ensure any previous spill is cleared
14494                        let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14495                        if let Some(d) = delta.as_deref_mut()
14496                            && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14497                        {
14498                            if spill_cells.is_empty() {
14499                                let old = self
14500                                    .read_cell_value(
14501                                        self.graph.sheet_name(anchor.sheet_id),
14502                                        anchor.coord.row() + 1,
14503                                        anchor.coord.col() + 1,
14504                                    )
14505                                    .unwrap_or(LiteralValue::Empty);
14506                                if old != other {
14507                                    d.record_cell(
14508                                        anchor.sheet_id,
14509                                        anchor.coord.row(),
14510                                        anchor.coord.col(),
14511                                    );
14512                                }
14513                            } else {
14514                                for cell in spill_cells.iter() {
14515                                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
14516                                    let old = self
14517                                        .read_cell_value(
14518                                            sheet_name,
14519                                            cell.coord.row() + 1,
14520                                            cell.coord.col() + 1,
14521                                        )
14522                                        .unwrap_or(LiteralValue::Empty);
14523                                    let new = if cell.sheet_id == anchor.sheet_id
14524                                        && cell.coord.row() == anchor.coord.row()
14525                                        && cell.coord.col() == anchor.coord.col()
14526                                    {
14527                                        other.clone()
14528                                    } else {
14529                                        LiteralValue::Empty
14530                                    };
14531                                    Self::record_cell_if_changed(d, cell, &old, &new);
14532                                }
14533                            }
14534                        }
14535                        self.graph.clear_spill_region(vertex_id);
14536                        if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14537                            self.record_structural_change(scope);
14538                        }
14539                        if self.config.arrow_storage_enabled
14540                            && self.config.delta_overlay_enabled
14541                            && self.config.write_formula_overlay_enabled
14542                        {
14543                            let empty = LiteralValue::Empty;
14544                            for cell in spill_cells.iter() {
14545                                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14546                                self.mirror_value_to_computed_overlay(
14547                                    &sheet_name,
14548                                    cell.coord.row() + 1,
14549                                    cell.coord.col() + 1,
14550                                    &empty,
14551                                );
14552                            }
14553                        }
14554                        self.graph.update_vertex_value_ref(vertex_id, &other);
14555                        // Optionally mirror into Arrow overlay for Arrow-backed reads
14556                        if self.config.arrow_storage_enabled
14557                            && self.config.delta_overlay_enabled
14558                            && self.config.write_formula_overlay_enabled
14559                        {
14560                            let anchor = self
14561                                .graph
14562                                .get_cell_ref(vertex_id)
14563                                .expect("cell ref for vertex");
14564                            let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14565                            self.mirror_value_to_computed_overlay(
14566                                &sheet_name,
14567                                anchor.coord.row() + 1,
14568                                anchor.coord.col() + 1,
14569                                &other,
14570                            );
14571                        }
14572                        Ok(other)
14573                    }
14574                }
14575            }
14576            Err(e) => {
14577                // Runtime Excel error: store as a cell value instead of propagating
14578                // as an exception so bulk eval paths don't fail the whole pass.
14579                let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14580                let err_val = LiteralValue::Error(e.clone());
14581                if let Some(d) = delta
14582                    && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14583                {
14584                    if spill_cells.is_empty() {
14585                        let old = self
14586                            .read_cell_value(
14587                                self.graph.sheet_name(anchor.sheet_id),
14588                                anchor.coord.row() + 1,
14589                                anchor.coord.col() + 1,
14590                            )
14591                            .unwrap_or(LiteralValue::Empty);
14592                        if old != err_val {
14593                            d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14594                        }
14595                    } else {
14596                        for cell in spill_cells.iter() {
14597                            let sheet_name = self.graph.sheet_name(cell.sheet_id);
14598                            let old = self
14599                                .read_cell_value(
14600                                    sheet_name,
14601                                    cell.coord.row() + 1,
14602                                    cell.coord.col() + 1,
14603                                )
14604                                .unwrap_or(LiteralValue::Empty);
14605                            let new = if cell.sheet_id == anchor.sheet_id
14606                                && cell.coord.row() == anchor.coord.row()
14607                                && cell.coord.col() == anchor.coord.col()
14608                            {
14609                                err_val.clone()
14610                            } else {
14611                                LiteralValue::Empty
14612                            };
14613                            Self::record_cell_if_changed(d, cell, &old, &new);
14614                        }
14615                    }
14616                }
14617                self.graph.clear_spill_region(vertex_id);
14618                if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14619                    self.record_structural_change(scope);
14620                }
14621                if self.config.arrow_storage_enabled
14622                    && self.config.delta_overlay_enabled
14623                    && self.config.write_formula_overlay_enabled
14624                {
14625                    let empty = LiteralValue::Empty;
14626                    for cell in spill_cells.iter() {
14627                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14628                        self.mirror_value_to_computed_overlay(
14629                            &sheet_name,
14630                            cell.coord.row() + 1,
14631                            cell.coord.col() + 1,
14632                            &empty,
14633                        );
14634                    }
14635                }
14636                self.graph.update_vertex_value_ref(vertex_id, &err_val);
14637                if self.config.arrow_storage_enabled
14638                    && self.config.delta_overlay_enabled
14639                    && self.config.write_formula_overlay_enabled
14640                {
14641                    let anchor = self
14642                        .graph
14643                        .get_cell_ref(vertex_id)
14644                        .expect("cell ref for vertex");
14645                    let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14646                    self.mirror_value_to_computed_overlay(
14647                        &sheet_name,
14648                        anchor.coord.row() + 1,
14649                        anchor.coord.col() + 1,
14650                        &err_val,
14651                    );
14652                }
14653                Ok(err_val)
14654            }
14655        }
14656    }
14657
14658    fn evaluate_named_scalar(
14659        &mut self,
14660        vertex_id: VertexId,
14661        sheet_id: SheetId,
14662    ) -> Result<LiteralValue, ExcelError> {
14663        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14664            ExcelError::new(ExcelErrorKind::Name)
14665                .with_message("Named range metadata missing".to_string())
14666        })?;
14667
14668        match &named_range.definition {
14669            NamedDefinition::Cell(cell_ref) => {
14670                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14671                let row = cell_ref.coord.row() + 1;
14672                let col = cell_ref.coord.col() + 1;
14673
14674                if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
14675                    && matches!(
14676                        self.graph.get_vertex_kind(dep_vertex),
14677                        VertexKind::FormulaScalar | VertexKind::FormulaArray
14678                    )
14679                {
14680                    // Graph does not cache cell/formula values; ensure the precedent is evaluated.
14681                    let value = self.evaluate_vertex(dep_vertex)?;
14682                    self.graph.update_vertex_value_ref(vertex_id, &value);
14683                    Ok(value)
14684                } else {
14685                    let value = self
14686                        .read_cell_value(sheet_name, row, col)
14687                        .unwrap_or(LiteralValue::Empty);
14688                    self.graph.update_vertex_value_ref(vertex_id, &value);
14689                    Ok(value)
14690                }
14691            }
14692            NamedDefinition::Literal(v) => {
14693                let out = v.clone();
14694                self.graph.update_vertex_value_ref(vertex_id, &out);
14695                Ok(out)
14696            }
14697            NamedDefinition::Formula { ast, .. } => {
14698                let context_sheet = match named_range.scope {
14699                    NameScope::Sheet(id) => id,
14700                    NameScope::Workbook => sheet_id,
14701                };
14702                let sheet_name = self.graph.sheet_name(context_sheet);
14703                let cell_ref = self
14704                    .graph
14705                    .get_cell_ref(vertex_id)
14706                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14707                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14708                    .with_resolved_functions(&self.resolved_functions)
14709                    .with_arena_cells(self);
14710                match interpreter.evaluate_ast(ast) {
14711                    Ok(cv) => {
14712                        let value = cv.into_literal();
14713                        match value {
14714                            LiteralValue::Array(_) => {
14715                                let err = ExcelError::new(ExcelErrorKind::NImpl)
14716                                    .with_message("Array result in scalar named range".to_string());
14717                                let err_val = LiteralValue::Error(err.clone());
14718                                self.graph.update_vertex_value_ref(vertex_id, &err_val);
14719                                Ok(err_val)
14720                            }
14721                            other => {
14722                                self.graph.update_vertex_value_ref(vertex_id, &other);
14723                                Ok(other)
14724                            }
14725                        }
14726                    }
14727                    Err(err) => {
14728                        let err_val = LiteralValue::Error(err.clone());
14729                        self.graph.update_vertex_value_ref(vertex_id, &err_val);
14730                        Ok(err_val)
14731                    }
14732                }
14733            }
14734            NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
14735                .with_message("Range-valued name evaluated as scalar".to_string())),
14736        }
14737    }
14738
14739    fn evaluate_named_array(
14740        &mut self,
14741        vertex_id: VertexId,
14742        sheet_id: SheetId,
14743    ) -> Result<LiteralValue, ExcelError> {
14744        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14745            ExcelError::new(ExcelErrorKind::Name)
14746                .with_message("Named range metadata missing".to_string())
14747        })?;
14748
14749        let out = match &named_range.definition {
14750            NamedDefinition::Range(range_ref) => {
14751                if range_ref.start.sheet_id != range_ref.end.sheet_id {
14752                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14753                        .with_message("Named range cannot span sheets".to_string()));
14754                }
14755
14756                let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
14757                let sr0 = range_ref.start.coord.row();
14758                let sc0 = range_ref.start.coord.col();
14759                let er0 = range_ref.end.coord.row();
14760                let ec0 = range_ref.end.coord.col();
14761                if sr0 > er0 || sc0 > ec0 {
14762                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14763                        .with_message("Invalid named range bounds".to_string()));
14764                }
14765
14766                let h = (er0 - sr0 + 1) as usize;
14767                let w = (ec0 - sc0 + 1) as usize;
14768                let cell_count = (h as u64).saturating_mul(w as u64);
14769                if cell_count > self.config.spill.max_spill_cells as u64 {
14770                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
14771                        "Named range too large to materialize as an array".to_string(),
14772                    ));
14773                }
14774
14775                let mut rows = Vec::with_capacity(h);
14776                for r0 in sr0..=er0 {
14777                    let mut row = Vec::with_capacity(w);
14778                    for c0 in sc0..=ec0 {
14779                        let v = self
14780                            .read_cell_value(sheet_name, r0 + 1, c0 + 1)
14781                            .unwrap_or(LiteralValue::Empty);
14782                        row.push(v);
14783                    }
14784                    rows.push(row);
14785                }
14786                LiteralValue::Array(rows)
14787            }
14788            NamedDefinition::Cell(cell_ref) => {
14789                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14790                let row = cell_ref.coord.row() + 1;
14791                let col = cell_ref.coord.col() + 1;
14792                let v = self
14793                    .read_cell_value(sheet_name, row, col)
14794                    .unwrap_or(LiteralValue::Empty);
14795                LiteralValue::Array(vec![vec![v]])
14796            }
14797            NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
14798            NamedDefinition::Formula { ast, .. } => {
14799                let context_sheet = match named_range.scope {
14800                    NameScope::Sheet(id) => id,
14801                    NameScope::Workbook => sheet_id,
14802                };
14803                let sheet_name = self.graph.sheet_name(context_sheet);
14804                let cell_ref = self
14805                    .graph
14806                    .get_cell_ref(vertex_id)
14807                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14808                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14809                    .with_resolved_functions(&self.resolved_functions)
14810                    .with_arena_cells(self);
14811                match interpreter.evaluate_ast(ast) {
14812                    Ok(cv) => {
14813                        let v = cv.into_literal();
14814                        match v {
14815                            LiteralValue::Array(_) => v,
14816                            other => LiteralValue::Array(vec![vec![other]]),
14817                        }
14818                    }
14819                    Err(err) => LiteralValue::Error(err),
14820                }
14821            }
14822        };
14823
14824        self.graph.update_vertex_value_ref(vertex_id, &out);
14825        Ok(out)
14826    }
14827
14828    fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
14829        crate::engine::ResourceLedgerError::Exhausted(
14830            formualizer_common::ResourceExhaustionDetail {
14831                reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
14832                limit: limit as u64,
14833                observed: limit.saturating_add(1) as u64,
14834                request_id: self
14835                    .active_evaluation_resource_request
14836                    .as_ref()
14837                    .map(|stats| stats.request_id),
14838            },
14839        )
14840        .into_excel_error()
14841        .with_message(format!("{context} did not converge after {limit} replans"))
14842    }
14843
14844    fn transient_target_preparation_stale(error: &ExcelError) -> bool {
14845        matches!(
14846            error.extra(),
14847            formualizer_common::ExcelErrorExtra::PreparationStale {
14848                reason: formualizer_common::PreparationStaleReason::Semantic
14849                    | formualizer_common::PreparationStaleReason::Provider
14850            }
14851        )
14852    }
14853
14854    fn prepare_graph_for_routed_evaluation(
14855        &mut self,
14856        targets: &[crate::engine::EvaluationTarget],
14857        options: &crate::engine::TargetEvalOptions<'_>,
14858    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
14859        const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
14860        let mut retries = 0usize;
14861        loop {
14862            match self.prepare_graph_for_targets_unobserved(targets, options) {
14863                Err(error)
14864                    if Self::transient_target_preparation_stale(&error)
14865                        && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
14866                {
14867                    retries = retries.saturating_add(1);
14868                }
14869                result => return result,
14870            }
14871        }
14872    }
14873
14874    fn evaluate_mixed_targets(
14875        &mut self,
14876        targets: &[crate::engine::EvaluationTarget],
14877        delta: Option<&mut DeltaCollector>,
14878    ) -> Result<EvalResult, ExcelError> {
14879        let _source_cache = self.source_cache_session();
14880        let cancel = self.active_cancel_flag.clone();
14881        let options = crate::engine::TargetEvalOptions {
14882            request_id: self
14883                .active_evaluation_resource_request
14884                .as_ref()
14885                .map(|stats| stats.request_id),
14886            cancel,
14887            deadline: None,
14888            budgets: None,
14889            opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
14890        };
14891        self.prepare_and_execute_target_recipe(targets, &options, delta)
14892    }
14893
14894    fn prepare_and_execute_target_recipe(
14895        &mut self,
14896        targets: &[crate::engine::EvaluationTarget],
14897        options: &crate::engine::TargetEvalOptions<'_>,
14898        delta: Option<&mut DeltaCollector>,
14899    ) -> Result<EvalResult, ExcelError> {
14900        let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
14901        self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
14902    }
14903
14904    fn execute_prepared_target_recipe(
14905        &mut self,
14906        targets: &[crate::engine::EvaluationTarget],
14907        scope: &crate::engine::PrepareScope,
14908        delta: Option<&mut DeltaCollector>,
14909    ) -> Result<EvalResult, ExcelError> {
14910        self.require_unified_authority()?;
14911        if matches!(scope, crate::engine::PrepareScope::Workbook)
14912            && let Some(stats) = self.active_evaluation_resource_request.as_mut()
14913        {
14914            stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
14915        }
14916        let mut roots = self.resolve_target_producers(targets)?;
14917        if let crate::engine::PrepareScope::Sheets(sheets) = scope {
14918            let request_id = self
14919                .active_evaluation_resource_request
14920                .as_ref()
14921                .map(|request| request.request_id);
14922            let root_count = roots.len();
14923            let mut widened_roots =
14924                OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
14925                    .map_err(|_| target_root_allocation_error(root_count, request_id))?;
14926            let sheet_ids = sheets
14927                .iter()
14928                .filter_map(|sheet| self.graph.sheet_id(sheet))
14929                .collect::<FxHashSet<_>>();
14930            for vertex in self.graph.formula_vertices() {
14931                if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
14932                    widened_roots
14933                        .push(crate::engine::target_preparation::TargetProducer::Legacy(
14934                            vertex,
14935                        ))
14936                        .map_err(|_| {
14937                            target_root_allocation_error(widened_roots.len() + 1, request_id)
14938                        })?;
14939                }
14940            }
14941            roots = widened_roots.into_vec();
14942        }
14943        self.begin_evaluation_request();
14944        #[cfg(any(test, feature = "legacy_oracle"))]
14945        self.graph.flush_pending_edge_deltas();
14946        let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
14947        if workbook_scope {
14948            if let Some(delta) = delta {
14949                self.evaluate_all_with_delta_collector(delta)
14950            } else {
14951                self.evaluate_all_legacy_impl()
14952            }
14953        } else {
14954            self.evaluate_legacy_target_roots(&roots, delta)
14955        }
14956    }
14957
14958    fn legacy_coordinate_targets(
14959        &mut self,
14960        targets: &[(&str, u32, u32)],
14961    ) -> Vec<crate::engine::EvaluationTarget> {
14962        targets
14963            .iter()
14964            .map(|(sheet, row, col)| {
14965                // Compatibility APIs historically interned an unknown target sheet
14966                // and returned an empty value rather than rejecting the target.
14967                self.graph.sheet_id_mut(sheet);
14968                crate::engine::EvaluationTarget::Cell {
14969                    sheet: (*sheet).to_string(),
14970                    row: *row,
14971                    col: *col,
14972                }
14973            })
14974            .collect()
14975    }
14976
14977    /// Evaluate the necessary mixed producer closure for typed cell, range, name, and table targets.
14978    pub fn evaluate_targets(
14979        &mut self,
14980        targets: &[crate::engine::EvaluationTarget],
14981    ) -> Result<EvalResult, ExcelError> {
14982        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14983            engine.observe_function_semantic_epoch()?;
14984            engine.validate_deterministic_mode()?;
14985            engine.evaluate_mixed_targets(targets, None)
14986        })
14987    }
14988
14989    /// Evaluate typed targets with explicit preparation policy and request controls.
14990    pub fn evaluate_targets_with_options(
14991        &mut self,
14992        targets: &[crate::engine::EvaluationTarget],
14993        options: crate::engine::TargetEvalOptions<'_>,
14994    ) -> Result<EvalResult, ExcelError> {
14995        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14996            engine.active_cancel_flag = options.cancel.clone();
14997            engine.active_evaluation_deadline = options.deadline;
14998            let result = (|| {
14999                engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
15000                engine.observe_function_semantic_epoch()?;
15001                engine.validate_deterministic_mode()?;
15002                let _source_cache = engine.source_cache_session();
15003                engine.prepare_and_execute_target_recipe(targets, &options, None)
15004            })();
15005            engine.active_cancel_flag = None;
15006            engine.active_evaluation_deadline = None;
15007            result
15008        })
15009    }
15010
15011    /// Evaluate typed targets and return the versioned run/region delta for the request.
15012    pub fn evaluate_targets_with_delta(
15013        &mut self,
15014        targets: &[crate::engine::EvaluationTarget],
15015    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15016        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15017            engine.observe_function_semantic_epoch()?;
15018            engine.validate_deterministic_mode()?;
15019            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15020            let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
15021            Ok((result, collector.finish_target()))
15022        })
15023    }
15024
15025    /// Evaluate only the necessary precedents for specific target cells (demand-driven)
15026    pub fn evaluate_until(
15027        &mut self,
15028        targets: &[(&str, u32, u32)],
15029    ) -> Result<EvalResult, ExcelError> {
15030        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
15031            engine.evaluate_until_unobserved(targets)
15032        })
15033    }
15034
15035    fn evaluate_until_unobserved(
15036        &mut self,
15037        targets: &[(&str, u32, u32)],
15038    ) -> Result<EvalResult, ExcelError> {
15039        self.observe_function_semantic_epoch()?;
15040        let targets = self.legacy_coordinate_targets(targets);
15041        self.evaluate_mixed_targets(&targets, None)
15042    }
15043
15044    fn evaluate_until_with_delta_collector(
15045        &mut self,
15046        targets: &[(&str, u32, u32)],
15047        delta: &mut DeltaCollector,
15048    ) -> Result<EvalResult, ExcelError> {
15049        let targets = self.legacy_coordinate_targets(targets);
15050        self.evaluate_mixed_targets(&targets, Some(delta))
15051    }
15052
15053    fn evaluate_legacy_target_roots(
15054        &mut self,
15055        roots: &[crate::engine::target_preparation::TargetProducer],
15056        mut delta: Option<&mut DeltaCollector>,
15057    ) -> Result<EvalResult, ExcelError> {
15058        use crate::engine::target_preparation::TargetProducer;
15059        #[cfg(any(test, feature = "benchmark_internal"))]
15060        {
15061            self.recalc_reuse_probe
15062                .get_mut()
15063                .unwrap()
15064                .legacy_target_requests += 1;
15065        }
15066        let start = crate::instant::FzInstant::now();
15067        let root_vertices = roots
15068            .iter()
15069            .filter_map(|root| match root {
15070                TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
15071                TargetProducer::ValueOnly(_) => None,
15072            })
15073            .collect::<Vec<_>>();
15074        let mut computed_vertices = 0usize;
15075        let mut cycle_errors = 0usize;
15076        let mut replans = 0usize;
15077        const MAX_REPLAN: usize = 5;
15078        self.graph.authority_sync();
15079        loop {
15080            let (precedents_to_eval, old_vdeps) = self.demand_subgraph(&root_vertices)?;
15081            if precedents_to_eval.is_empty() {
15082                break;
15083            }
15084            #[cfg(any(test, feature = "benchmark_internal"))]
15085            {
15086                self.recalc_reuse_probe
15087                    .get_mut()
15088                    .unwrap()
15089                    .target_schedule_builds += 1;
15090            }
15091            let schedule = {
15092                self.graph.authority_sync();
15093                let mut ledger = self.active_resource_ledger.take();
15094                let result = self.create_authority_schedule(
15095                    &precedents_to_eval,
15096                    &old_vdeps,
15097                    ledger.as_mut(),
15098                );
15099                self.active_resource_ledger = ledger;
15100                result?
15101            };
15102            self.begin_pass(&schedule);
15103            for (unit_index, &unit) in schedule.units.iter().enumerate() {
15104                self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
15105                match unit {
15106                    ScheduleUnit::Cycle(index) => {
15107                        if self.handle_cycle_unit(
15108                            schedule.unit_cycle(index),
15109                            delta.as_deref_mut(),
15110                            None,
15111                            None,
15112                        )? > 0
15113                        {
15114                            cycle_errors = cycle_errors.saturating_add(1);
15115                        }
15116                    }
15117                    ScheduleUnit::Layer(index) => {
15118                        let layer = schedule.unit_layer(index);
15119                        let evaluated = if let Some(delta) = delta.as_deref_mut() {
15120                            if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15121                                self.evaluate_layer_parallel_with_delta(layer, delta)?
15122                            } else {
15123                                self.evaluate_layer_sequential_with_delta(layer, delta)?
15124                            }
15125                        } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15126                            self.evaluate_layer_parallel(layer)?
15127                        } else {
15128                            self.evaluate_layer_sequential(layer)?
15129                        };
15130                        computed_vertices = computed_vertices.saturating_add(evaluated);
15131                    }
15132                }
15133                if self.stop_after_unit(&schedule, unit_index) {
15134                    break;
15135                }
15136            }
15137            let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
15138            self.resource_checkpoint(0)?;
15139            if !self.finish_target_pass_dirty(&precedents_to_eval, &changed) {
15140                break;
15141            }
15142            if replans >= MAX_REPLAN {
15143                return Err(self.replan_exhausted_error(
15144                    MAX_REPLAN,
15145                    "targeted legacy dynamic dependency evaluation",
15146                ));
15147            }
15148            replans = replans.saturating_add(1);
15149        }
15150        self.redirty_for_next_recalc();
15151        Ok(EvalResult {
15152            computed_vertices,
15153            cycle_errors,
15154            elapsed: start.elapsed(),
15155        })
15156    }
15157
15158    /// Build a revision-bound compatibility plan covering every prepared formula vertex.
15159    pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
15160        if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
15161            return Err(
15162                Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
15163                    "compatibility recalculation plans require all staged formulas to be prepared",
15164                ),
15165            );
15166        }
15167        let key = self.recalc_plan_key();
15168        let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
15169        vertices.sort_unstable();
15170        let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
15171        let schedule = if vertices.is_empty() {
15172            crate::engine::Schedule {
15173                units: Vec::new(),
15174                layers: Vec::new(),
15175                cycles: Vec::new(),
15176            }
15177        } else {
15178            self.create_evaluation_schedule_uncached(&vertices, None)?.0
15179        };
15180        self.validate_recalc_plan_key(&key)?;
15181        Ok(RecalcPlan {
15182            key,
15183            kind: RecalcPlanKind::CompatibilityFull {
15184                schedule,
15185                has_dynamic_refs,
15186            },
15187        })
15188    }
15189
15190    /// Prepare stable typed targets and retain a revision-bound run-local recipe.
15191    pub fn build_recalc_plan_for_targets(
15192        &mut self,
15193        targets: &[crate::engine::EvaluationTarget],
15194    ) -> Result<RecalcPlan, ExcelError> {
15195        self.build_recalc_plan_for_targets_with_options(
15196            targets,
15197            crate::engine::TargetEvalOptions::default(),
15198        )
15199    }
15200
15201    pub fn build_recalc_plan_for_targets_with_options(
15202        &mut self,
15203        targets: &[crate::engine::EvaluationTarget],
15204        options: crate::engine::TargetEvalOptions<'_>,
15205    ) -> Result<RecalcPlan, ExcelError> {
15206        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15207            engine.observe_function_semantic_epoch()?;
15208            engine.validate_deterministic_mode()?;
15209            let _source_cache = engine.source_cache_session();
15210            let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
15211            #[cfg(any(test, feature = "legacy_oracle"))]
15212            engine.graph.flush_pending_edge_deltas();
15213            let topology = if matches!(
15214                preparation.widened_scope,
15215                crate::engine::PrepareScope::Workbook
15216            ) {
15217                RecalcTopology::Workbook
15218            } else {
15219                RecalcTopology::RunLocalRecipe
15220            };
15221            Ok(RecalcPlan {
15222                key: engine.recalc_plan_key(),
15223                kind: RecalcPlanKind::Target {
15224                    targets: targets.to_vec(),
15225                    scope: preparation.widened_scope,
15226                    topology,
15227                    dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
15228                },
15229            })
15230        })
15231    }
15232
15233    /// Evaluate using a previously constructed compatibility or target plan.
15234    pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
15235        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15236            engine.evaluate_recalc_plan_unobserved(plan)
15237        })
15238    }
15239
15240    pub fn evaluate_recalc_plan_with_controls(
15241        &mut self,
15242        plan: &RecalcPlan,
15243        cancel: Option<crate::engine::CancelToken>,
15244        deadline: Option<Instant>,
15245    ) -> Result<EvalResult, ExcelError> {
15246        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15247            engine.active_cancel_flag = cancel.clone();
15248            engine.active_evaluation_deadline = deadline;
15249            let result = engine.evaluate_recalc_plan_unobserved(plan);
15250            engine.active_cancel_flag = None;
15251            engine.active_evaluation_deadline = None;
15252            result
15253        })
15254    }
15255
15256    fn evaluate_recalc_plan_unobserved(
15257        &mut self,
15258        plan: &RecalcPlan,
15259    ) -> Result<EvalResult, ExcelError> {
15260        #[cfg(any(test, feature = "legacy_oracle"))]
15261        self.graph.flush_pending_edge_deltas();
15262        self.validate_recalc_plan_key(&plan.key)?;
15263        self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
15264        self.validate_deterministic_mode()?;
15265
15266        match &plan.kind {
15267            RecalcPlanKind::Target {
15268                targets,
15269                scope,
15270                topology,
15271                dynamic_policy,
15272            } => {
15273                debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
15274                debug_assert_eq!(
15275                    matches!(topology, RecalcTopology::Workbook),
15276                    matches!(scope, crate::engine::PrepareScope::Workbook)
15277                );
15278                let _source_cache = self.source_cache_session();
15279                self.execute_prepared_target_recipe(targets, scope, None)
15280            }
15281            RecalcPlanKind::CompatibilityFull {
15282                schedule,
15283                has_dynamic_refs,
15284            } => {
15285                let _source_cache = self.source_cache_session();
15286                self.begin_evaluation_request();
15287                if *has_dynamic_refs {
15288                    self.virtual_dep_fallback_activations =
15289                        self.virtual_dep_fallback_activations.saturating_add(1);
15290                    return self.evaluate_all_coordinator();
15291                }
15292
15293                let start = crate::instant::FzInstant::now();
15294                let dirty_vertices = self.graph.get_evaluation_vertices();
15295                if dirty_vertices.is_empty() {
15296                    return Ok(EvalResult {
15297                        computed_vertices: 0,
15298                        cycle_errors: 0,
15299                        elapsed: start.elapsed(),
15300                    });
15301                }
15302
15303                let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
15304                let mut computed_vertices = 0;
15305                let mut cycle_errors = 0;
15306                for &unit in &schedule.units {
15307                    self.cancellation_checkpoint(
15308                        "Evaluation cancelled before recalculation plan schedule unit",
15309                    )?;
15310                    match unit {
15311                        ScheduleUnit::Cycle(i) => {
15312                            let stamped = self.handle_cycle_unit(
15313                                schedule.unit_cycle(i),
15314                                None,
15315                                Some(&dirty_set),
15316                                None,
15317                            )?;
15318                            if stamped > 0 {
15319                                cycle_errors += 1;
15320                            }
15321                        }
15322                        ScheduleUnit::Layer(i) => {
15323                            let work: Vec<VertexId> = schedule
15324                                .unit_layer(i)
15325                                .vertices
15326                                .iter()
15327                                .copied()
15328                                .filter(|v| dirty_set.contains(v))
15329                                .collect();
15330                            if work.is_empty() {
15331                                continue;
15332                            }
15333                            let temp_layer = crate::engine::scheduler::Layer::new(work);
15334                            if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
15335                                computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
15336                            } else {
15337                                computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
15338                            }
15339                        }
15340                    }
15341                }
15342
15343                self.resource_checkpoint(0)?;
15344                self.graph.clear_dirty_flags(&dirty_vertices);
15345                self.redirty_for_next_recalc();
15346                Ok(EvalResult {
15347                    computed_vertices,
15348                    cycle_errors,
15349                    elapsed: start.elapsed(),
15350                })
15351            }
15352        }
15353    }
15354}
15355
15356impl<R> Engine<R>
15357where
15358    R: EvaluationContext,
15359{
15360    /// Refuse out-of-scope authority states before evaluation can demote spans
15361    /// or execute a legacy schedule. The public error type is unchanged; NImpl
15362    /// carries the exact internal Unsupported operation for the deferred-scope
15363    /// gate. Admission and allocation failures are not scope exceptions.
15364    fn require_unified_authority(&mut self) -> Result<(), ExcelError> {
15365        self.graph
15366            .authority()
15367            .map(|_| ())
15368            .map_err(Self::authority_excel_error)
15369    }
15370
15371    fn authority_excel_error(error: crate::engine::authority::store::AuthorityError) -> ExcelError {
15372        use crate::engine::authority::store::AuthorityError;
15373        // Some unchanged behavioral tests assert only `error.kind`, hiding
15374        // the operation in their panic. The opt-in gate trace proves which
15375        // typed error was actually returned; it never changes that error.
15376        #[cfg(test)]
15377        if std::env::var_os("FZ_AUTHORITY_DEFERRED_TRACE").is_some() {
15378            eprintln!("M1B_AUTHORITY_ERROR {error:?}");
15379        }
15380        let kind = match error {
15381            AuthorityError::Unsupported { .. } => ExcelErrorKind::NImpl,
15382            _ => ExcelErrorKind::Error,
15383        };
15384        ExcelError::new(kind).with_message(format!("unified_authority: {error:?}"))
15385    }
15386
15387    /// Evaluate all dirty/volatile vertices
15388    /// Evaluate a current result snapshot, optionally with cancellation.
15389    ///
15390    /// This evaluates exactly like [`Self::evaluate_all`] (or its cancellable
15391    /// counterpart), but leaves volatile vertices and their dependents Current
15392    /// at return rather than marking them dirty for the next cycle. It does
15393    /// not suppress iterative-SCC redirty or any other freshness check.
15394    ///
15395    /// Callers retaining the engine must start another evaluation request
15396    /// (`evaluate_all`, `evaluate_until`, `evaluate_cell`, or equivalent) to
15397    /// refresh volatile values. That request restores the deferred volatile
15398    /// redirty before selecting its work. Errors and cancellation never leave
15399    /// the snapshot mode active.
15400    pub fn evaluate_all_for_snapshot(
15401        &mut self,
15402        cancel: Option<crate::engine::CancelToken>,
15403    ) -> Result<EvalResult, ExcelError> {
15404        /// Leaves snapshot mode even when evaluation unwinds. A panic
15405        /// keeps the deferred volatile redirty pending, so the next request
15406        /// restores volatile work without touching the graph mid-unwind.
15407        struct SnapshotScope<'e, R: EvaluationContext>(&'e mut Engine<R>);
15408        impl<R: EvaluationContext> Drop for SnapshotScope<'_, R> {
15409            fn drop(&mut self) {
15410                self.0.snapshot_evaluation_active = false;
15411                if std::thread::panicking() {
15412                    self.0.snapshot_volatile_redirty_pending = true;
15413                }
15414            }
15415        }
15416        self.snapshot_evaluation_active = true;
15417        let result = {
15418            let scope = SnapshotScope(self);
15419            match cancel {
15420                Some(cancel) => scope.0.evaluate_all_cancellable(cancel),
15421                None => scope.0.evaluate_all(),
15422            }
15423        };
15424        if result.is_err() {
15425            self.snapshot_volatile_redirty_pending = false;
15426            self.graph.redirty_volatiles();
15427        }
15428        result
15429    }
15430
15431    /// Evaluate all dirty/volatile vertices.
15432    pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
15433        // `evaluate_all_unobserved` owns the `observe_function_semantic_epoch` guard.
15434        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
15435            engine.evaluate_all_unobserved()
15436        })
15437    }
15438
15439    fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
15440        debug_assert!(
15441            !self.graph.deferred_dirty_active(),
15442            "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
15443             was not balanced by end_deferred_dirty"
15444        );
15445        self.observe_function_semantic_epoch()?;
15446        self.lookup_index_cache.reset_counters();
15447        let _source_cache = self.source_cache_session();
15448        self.validate_deterministic_mode()?;
15449        if self.config.defer_graph_building {
15450            // Build graph for all staged formulas before evaluating
15451            self.build_graph_all()?;
15452        }
15453        self.evaluate_all_coordinator()
15454    }
15455
15456    /// Coordinator for `evaluate_all`: starts the evaluation request and runs
15457    /// the per-cell pass.
15458    fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
15459        self.require_unified_authority()?;
15460        self.begin_evaluation_request();
15461        self.evaluate_all_legacy_impl()
15462    }
15463
15464    /// Walk a schedule's units in condensation order: stamp each cyclic SCC
15465    /// at its position and evaluate each layer (parallel when enabled).
15466    ///
15467    /// Returns `(computed_vertices, cycle_count)` where `cycle_count` is the
15468    /// number of Cycle units walked (the former `schedule.cycles.len()`).
15469    fn legacy_pass_run_units(
15470        &mut self,
15471        schedule: &crate::engine::scheduler::Schedule,
15472    ) -> Result<(usize, usize), ExcelError> {
15473        let mut computed_vertices = 0;
15474        let mut cycle_count = 0;
15475        self.begin_pass(schedule);
15476        for (unit_index, &unit) in schedule.units.iter().enumerate() {
15477            match unit {
15478                ScheduleUnit::Cycle(i) => {
15479                    if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
15480                        cycle_count += 1;
15481                    }
15482                }
15483                ScheduleUnit::Layer(i) => {
15484                    let layer = schedule.unit_layer(i);
15485                    if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15486                        computed_vertices += self.evaluate_layer_parallel(layer)?;
15487                    } else {
15488                        computed_vertices += self.evaluate_layer_sequential(layer)?;
15489                    }
15490                }
15491            }
15492            if self.stop_after_unit(schedule, unit_index) {
15493                break;
15494            }
15495        }
15496        Ok((computed_vertices, cycle_count))
15497    }
15498
15499    /// Per-cell `evaluate_all` body, reached through the coordinator. This is
15500    /// an internal primitive; it must not be invoked directly from public APIs.
15501    ///
15502    /// Does NOT call `begin_evaluation_request` (cycle-telemetry reset +
15503    /// per-recalc clock sample): request begin happens at the public entry
15504    /// points / coordinators, so one request keeps one clock sample.
15505    fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
15506        self.reset_virtual_dep_telemetry_if_disabled();
15507        let _span_eval =
15508            crate::engine::trace::fz_span!(tracing::Level::INFO, "evaluate", "evaluate.legacy");
15509        let start = crate::instant::FzInstant::now();
15510        let mut computed_vertices = 0;
15511        let mut cycle_errors = 0;
15512        let mut replan_iterations = 0;
15513        const MAX_REPLAN: usize = 5;
15514        let mut telemetry = self
15515            .config
15516            .enable_virtual_dep_telemetry
15517            .then(|| self.start_virtual_dep_telemetry());
15518
15519        loop {
15520            let to_evaluate = self.graph.get_evaluation_vertices();
15521            if to_evaluate.is_empty() {
15522                if let Some(t) = telemetry.as_mut()
15523                    && t.bailout_reason.is_none()
15524                {
15525                    t.bailout_reason = Some("no_work");
15526                }
15527                break;
15528            }
15529
15530            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15531            if let Some(t) = telemetry.as_mut() {
15532                Self::accumulate_schedule_meta(t, &meta);
15533            }
15534
15535            let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
15536            computed_vertices += pass_computed;
15537            cycle_errors += pass_cycles;
15538
15539            // Check if dynamic dependencies changed
15540            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15541            if let Some(t) = telemetry.as_mut() {
15542                t.changed_vdeps_total += changed_vertices.len();
15543            }
15544
15545            self.resource_checkpoint(0)?;
15546            if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15547                if let Some(t) = telemetry.as_mut() {
15548                    t.bailout_reason = Some("converged");
15549                }
15550                break;
15551            }
15552            if replan_iterations >= MAX_REPLAN {
15553                if let Some(mut t) = telemetry.take() {
15554                    t.bailout_reason = Some("max_replan");
15555                    t.replan_iterations = replan_iterations;
15556                    self.last_virtual_dep_telemetry = t;
15557                }
15558                return Err(
15559                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15560                );
15561            }
15562
15563            replan_iterations += 1;
15564        }
15565
15566        if let Some(mut t) = telemetry {
15567            t.replan_iterations = replan_iterations;
15568            self.last_virtual_dep_telemetry = t;
15569        }
15570
15571        // Re-dirty volatile vertices for the next evaluation cycle
15572        self.redirty_for_next_recalc();
15573
15574        // Advance recalc epoch after a full evaluation pass finishes
15575        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15576
15577        Ok(EvalResult {
15578            computed_vertices,
15579            cycle_errors,
15580            elapsed: start.elapsed(),
15581        })
15582    }
15583
15584    pub fn evaluate_all_with_target_delta(
15585        &mut self,
15586    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15587        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15588            engine.observe_function_semantic_epoch()?;
15589            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15590            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15591            Ok((result, collector.finish_target()))
15592        })
15593    }
15594
15595    pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
15596        self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
15597    }
15598
15599    pub fn evaluate_all_with_delta_policy(
15600        &mut self,
15601        policy: EvalDeltaCompatibilityPolicy,
15602    ) -> Result<(EvalResult, EvalDelta), ExcelError> {
15603        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15604            engine.observe_function_semantic_epoch()?;
15605            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15606            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15607            Ok((result, collector.finish_with_policy(policy)?))
15608        })
15609    }
15610
15611    fn evaluate_all_with_delta_collector(
15612        &mut self,
15613        delta: &mut DeltaCollector,
15614    ) -> Result<EvalResult, ExcelError> {
15615        let _source_cache = self.source_cache_session();
15616        if self.config.defer_graph_building {
15617            self.build_graph_all()?;
15618        }
15619        self.require_unified_authority()?;
15620        self.begin_evaluation_request();
15621        self.reset_virtual_dep_telemetry_if_disabled();
15622        let _span_eval = crate::engine::trace::fz_span!(
15623            tracing::Level::INFO,
15624            "evaluate",
15625            "evaluate.legacy_delta"
15626        );
15627        let start = crate::instant::FzInstant::now();
15628        let mut computed_vertices = 0;
15629        let mut cycle_errors = 0;
15630
15631        let mut replan_iterations = 0;
15632        const MAX_REPLAN: usize = 5;
15633        let mut telemetry = self
15634            .config
15635            .enable_virtual_dep_telemetry
15636            .then(|| self.start_virtual_dep_telemetry());
15637
15638        loop {
15639            let to_evaluate = self.graph.get_evaluation_vertices();
15640            if to_evaluate.is_empty() {
15641                if let Some(t) = telemetry.as_mut()
15642                    && t.bailout_reason.is_none()
15643                {
15644                    t.bailout_reason = Some("no_work");
15645                }
15646                break;
15647            }
15648
15649            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15650            if let Some(t) = telemetry.as_mut() {
15651                Self::accumulate_schedule_meta(t, &meta);
15652            }
15653
15654            self.begin_pass(&schedule);
15655            for (unit_index, &unit) in schedule.units.iter().enumerate() {
15656                match unit {
15657                    ScheduleUnit::Cycle(i) => {
15658                        if self.handle_cycle_unit(
15659                            schedule.unit_cycle(i),
15660                            Some(delta),
15661                            None,
15662                            None,
15663                        )? > 0
15664                        {
15665                            cycle_errors += 1;
15666                        }
15667                    }
15668                    ScheduleUnit::Layer(i) => {
15669                        let layer = schedule.unit_layer(i);
15670                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15671                            computed_vertices +=
15672                                self.evaluate_layer_parallel_with_delta(layer, delta)?;
15673                        } else {
15674                            computed_vertices +=
15675                                self.evaluate_layer_sequential_with_delta(layer, delta)?;
15676                        }
15677                    }
15678                }
15679                if self.stop_after_unit(&schedule, unit_index) {
15680                    break;
15681                }
15682            }
15683
15684            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15685            if let Some(t) = telemetry.as_mut() {
15686                t.changed_vdeps_total += changed_vertices.len();
15687            }
15688            self.resource_checkpoint(0)?;
15689            if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15690                if let Some(t) = telemetry.as_mut() {
15691                    t.bailout_reason = Some("converged");
15692                }
15693                break;
15694            }
15695            if replan_iterations >= MAX_REPLAN {
15696                if let Some(mut t) = telemetry.take() {
15697                    t.bailout_reason = Some("max_replan");
15698                    t.replan_iterations = replan_iterations;
15699                    self.last_virtual_dep_telemetry = t;
15700                }
15701                return Err(
15702                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15703                );
15704            }
15705            replan_iterations += 1;
15706        }
15707
15708        if let Some(mut t) = telemetry {
15709            t.replan_iterations = replan_iterations;
15710            self.last_virtual_dep_telemetry = t;
15711        }
15712
15713        self.redirty_for_next_recalc();
15714        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15715
15716        Ok(EvalResult {
15717            computed_vertices,
15718            cycle_errors,
15719            elapsed: start.elapsed(),
15720        })
15721    }
15722
15723    /// Convenience: demand-driven evaluation of a single cell by sheet name and row/col.
15724    ///
15725    /// This will evaluate only the minimal set of dirty / volatile precedents required
15726    /// to bring the target cell up-to-date (as if a user asked for that single value),
15727    /// rather than scheduling a full workbook recalc. If the cell is already clean and
15728    /// non-volatile, no vertices will be recomputed.
15729    ///
15730    /// Returns the (possibly newly computed) value stored for the cell afterwards.
15731    /// Empty cells return None. Errors are surfaced via the Result type.
15732    pub fn evaluate_cell(
15733        &mut self,
15734        sheet: &str,
15735        row: u32,
15736        col: u32,
15737    ) -> Result<Option<LiteralValue>, ExcelError> {
15738        self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
15739            engine.evaluate_cell_unobserved(sheet, row, col)
15740        })
15741    }
15742
15743    fn evaluate_cell_unobserved(
15744        &mut self,
15745        sheet: &str,
15746        row: u32,
15747        col: u32,
15748    ) -> Result<Option<LiteralValue>, ExcelError> {
15749        if row == 0 || col == 0 {
15750            return Err(ExcelError::new(ExcelErrorKind::Ref)
15751                .with_message("Row and column must be >= 1".to_string()));
15752        }
15753
15754        let result = self.evaluate_cells(&[(sheet, row, col)])?;
15755
15756        match result.len() {
15757            0 => Ok(None),
15758            1 => {
15759                let v = result.into_iter().next().unwrap();
15760                Ok(v)
15761            }
15762            _ => unreachable!("evaluate_cells returned unexpected length"),
15763        }
15764    }
15765
15766    /// Convenience: demand-driven evaluation of multiple cells; accepts a slice of
15767    /// (sheet, row, col) triples. The union of required dirty / volatile precedents
15768    /// is computed once and evaluated, which is typically faster than calling
15769    /// `evaluate_cell` repeatedly for a related set of targets.
15770    ///
15771    /// Returns the resulting values for each requested target in the same order.
15772    pub fn evaluate_cells(
15773        &mut self,
15774        targets: &[(&str, u32, u32)],
15775    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15776        self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
15777            engine.evaluate_cells_unobserved(targets)
15778        })
15779    }
15780
15781    fn evaluate_cells_unobserved(
15782        &mut self,
15783        targets: &[(&str, u32, u32)],
15784    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15785        self.observe_function_semantic_epoch()?;
15786        debug_assert!(
15787            !self.graph.deferred_dirty_active(),
15788            "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
15789             was not balanced by end_deferred_dirty"
15790        );
15791        self.validate_deterministic_mode()?;
15792        if targets.is_empty() {
15793            return Ok(Vec::new());
15794        }
15795        let typed_targets = self.legacy_coordinate_targets(targets);
15796        self.evaluate_mixed_targets(&typed_targets, None)?;
15797        Ok(targets
15798            .iter()
15799            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15800            .collect())
15801    }
15802
15803    pub fn evaluate_cells_cancellable(
15804        &mut self,
15805        targets: &[(&str, u32, u32)],
15806        cancel: crate::engine::CancelToken,
15807    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15808        self.observe_evaluation_resource_request(
15809            EvaluationRequestKind::CellsCancellable,
15810            |engine| {
15811                engine.observe_function_semantic_epoch()?;
15812                engine.active_cancel_flag = Some(cancel.clone());
15813                let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
15814                engine.active_cancel_flag = None;
15815                res
15816            },
15817        )
15818    }
15819
15820    fn evaluate_cells_cancellable_impl(
15821        &mut self,
15822        targets: &[(&str, u32, u32)],
15823        cancel_flag: &AtomicBool,
15824    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15825        self.validate_deterministic_mode()?;
15826        if targets.is_empty() {
15827            return Ok(Vec::new());
15828        }
15829        if cancel_flag.load(Ordering::Relaxed) {
15830            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
15831                .with_message("Evaluation cancelled before target preparation"));
15832        }
15833        let typed_targets = self.legacy_coordinate_targets(targets);
15834        self.evaluate_mixed_targets(&typed_targets, None)?;
15835        Ok(targets
15836            .iter()
15837            .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
15838            .collect())
15839    }
15840
15841    pub fn evaluate_cells_with_target_delta(
15842        &mut self,
15843        targets: &[(&str, u32, u32)],
15844    ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
15845        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15846            engine.observe_function_semantic_epoch()?;
15847            engine.validate_deterministic_mode()?;
15848            if targets.is_empty() {
15849                return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
15850            }
15851            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15852            engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
15853            let values = targets
15854                .iter()
15855                .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
15856                .collect();
15857            Ok((values, collector.finish_target()))
15858        })
15859    }
15860
15861    pub fn evaluate_cells_with_delta(
15862        &mut self,
15863        targets: &[(&str, u32, u32)],
15864    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15865        self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
15866    }
15867
15868    pub fn evaluate_cells_with_delta_policy(
15869        &mut self,
15870        targets: &[(&str, u32, u32)],
15871        policy: EvalDeltaCompatibilityPolicy,
15872    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15873        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15874            engine.evaluate_cells_with_delta_unobserved(targets, policy)
15875        })
15876    }
15877
15878    fn evaluate_cells_with_delta_unobserved(
15879        &mut self,
15880        targets: &[(&str, u32, u32)],
15881        policy: EvalDeltaCompatibilityPolicy,
15882    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15883        self.observe_function_semantic_epoch()?;
15884        self.validate_deterministic_mode()?;
15885        if targets.is_empty() {
15886            return Ok((Vec::new(), EvalDelta::default()));
15887        }
15888        let mut collector = DeltaCollector::new(DeltaMode::Cells);
15889        self.evaluate_until_with_delta_collector(targets, &mut collector)?;
15890        let values = targets
15891            .iter()
15892            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15893            .collect();
15894        Ok((values, collector.finish_with_policy(policy)?))
15895    }
15896
15897    /// Get the evaluation plan for target cells without actually evaluating them
15898    pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
15899        if targets.is_empty() {
15900            return Ok(EvalPlan {
15901                total_vertices_to_evaluate: 0,
15902                layers: Vec::new(),
15903                cycles_detected: 0,
15904                dirty_count: 0,
15905                volatile_count: 0,
15906                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15907                estimated_parallel_layers: 0,
15908                target_cells: Vec::new(),
15909            });
15910        }
15911        if self.config.defer_graph_building && self.has_staged_formulas() {
15912            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
15913                "Evaluation plan requested with deferred graph; build first or call evaluate_*",
15914            ));
15915        }
15916
15917        // Convert targets to A1 notation for consistency
15918        let addresses: Vec<String> = targets
15919            .iter()
15920            .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
15921            .collect();
15922
15923        // Parse target cell addresses
15924        let mut target_addrs = Vec::new();
15925        for (sheet, row, col) in targets {
15926            if let Some(sheet_id) = self.graph.sheet_id(sheet) {
15927                let coord = Coord::from_excel(*row, *col, true, true);
15928                target_addrs.push(CellRef::new(sheet_id, coord));
15929            }
15930        }
15931
15932        // Find vertex IDs for targets
15933        let mut target_vertex_ids = Vec::new();
15934        for addr in &target_addrs {
15935            if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
15936                target_vertex_ids.push(vertex_id);
15937            }
15938        }
15939
15940        if target_vertex_ids.is_empty() {
15941            return Ok(EvalPlan {
15942                total_vertices_to_evaluate: 0,
15943                layers: Vec::new(),
15944                cycles_detected: 0,
15945                dirty_count: 0,
15946                volatile_count: 0,
15947                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15948                estimated_parallel_layers: 0,
15949                target_cells: addresses,
15950            });
15951        }
15952
15953        // Build demand subgraph with virtual edges (same as evaluate_until)
15954        let (precedents_to_eval, vdeps) = self.demand_subgraph(&target_vertex_ids)?;
15955
15956        if precedents_to_eval.is_empty() {
15957            return Ok(EvalPlan {
15958                total_vertices_to_evaluate: 0,
15959                layers: Vec::new(),
15960                cycles_detected: 0,
15961                dirty_count: 0,
15962                volatile_count: 0,
15963                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15964                estimated_parallel_layers: 0,
15965                target_cells: addresses,
15966            });
15967        }
15968
15969        // Count dirty and volatile vertices
15970        let mut dirty_count = 0;
15971        let mut volatile_count = 0;
15972        for &vertex_id in &precedents_to_eval {
15973            if self.graph.is_dirty(vertex_id) {
15974                dirty_count += 1;
15975            }
15976            if self.graph.is_volatile(vertex_id) {
15977                volatile_count += 1;
15978            }
15979        }
15980
15981        // Create schedule for the minimal subgraph honoring virtual edges
15982        let schedule = self.create_authority_schedule(&precedents_to_eval, &vdeps, None)?;
15983
15984        // Build layer information
15985        let mut layers = Vec::new();
15986        let mut estimated_parallel_layers = 0;
15987        let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
15988
15989        for layer in &schedule.layers {
15990            let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
15991            if parallel_eligible {
15992                estimated_parallel_layers += 1;
15993            }
15994
15995            // Get sample cell addresses (up to 5)
15996            let sample_cells: Vec<String> = layer
15997                .vertices
15998                .iter()
15999                .take(5)
16000                .filter_map(|&vertex_id| {
16001                    self.graph
16002                        .get_cell_ref_for_vertex(vertex_id)
16003                        .map(|cell_ref| {
16004                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
16005                            format!(
16006                                "{}!{}{}",
16007                                sheet_name,
16008                                Self::col_to_letters(cell_ref.coord.col().saturating_add(1)),
16009                                cell_ref.coord.row() + 1
16010                            )
16011                        })
16012                })
16013                .collect();
16014
16015            layers.push(LayerInfo {
16016                vertex_count: layer.vertices.len(),
16017                parallel_eligible,
16018                sample_cells,
16019            });
16020        }
16021
16022        Ok(EvalPlan {
16023            total_vertices_to_evaluate: precedents_to_eval.len(),
16024            layers,
16025            cycles_detected: schedule.cycles.len(),
16026            dirty_count,
16027            volatile_count,
16028            parallel_enabled,
16029            estimated_parallel_layers,
16030            target_cells: addresses,
16031        })
16032    }
16033    /// Helper to create a schedule, integrating virtual dependencies automatically.
16034    fn create_evaluation_schedule(
16035        &mut self,
16036        to_evaluate: &[VertexId],
16037    ) -> Result<EvaluationScheduleBuildOutput, ExcelError> {
16038        #[cfg(any(test, feature = "benchmark_internal"))]
16039        {
16040            self.recalc_reuse_probe.get_mut().unwrap().schedule_requests += 1;
16041        }
16042        // Fold pending edge deltas once per schedule build so traversal uses
16043        // the zero-allocation CSR slices (#125).
16044        #[cfg(any(test, feature = "legacy_oracle"))]
16045        self.graph.flush_pending_edge_deltas();
16046        // The cache key includes the authority revision: sync first.
16047        self.graph.authority_sync();
16048        if self.can_use_static_schedule_cache(to_evaluate) {
16049            // A recent schedule for the same request becomes the current one.
16050            let revision = self.schedule_cache_authority_revision();
16051            let current = |e: &CachedScheduleEntry| {
16052                e.topology_epoch == self.topology_epoch && e.authority_revision == revision
16053            };
16054            if !self
16055                .cached_static_schedule
16056                .as_ref()
16057                .is_some_and(|c| current(c) && c.candidate_vertices.equals(to_evaluate))
16058                && let Some(i) = self.recent_schedules.iter().position(|e| {
16059                    current(e)
16060                        && e.candidate_vertices.len() == to_evaluate.len()
16061                        && e.candidate_vertices.equals(to_evaluate)
16062                })
16063            {
16064                let hit = self.recent_schedules.remove(i);
16065                if let Some(previous) = self.cached_static_schedule.replace(hit) {
16066                    self.retain_recent_schedule(previous);
16067                }
16068            }
16069            if let Some(cached) = self.cached_static_schedule.as_ref()
16070                && cached.topology_epoch == self.topology_epoch
16071                && cached.authority_revision == self.schedule_cache_authority_revision()
16072                && cached.candidate_vertices.equals(to_evaluate)
16073            {
16074                let meta = ScheduleBuildMeta {
16075                    candidate_vertices: to_evaluate.len(),
16076                    vdeps_vertices: 0,
16077                    vdeps_edges: 0,
16078                    builder_elapsed_ms: 0,
16079                    used_virtual_schedule: false,
16080                    schedule_cache_hit: true,
16081                    schedule_cache_eligible: true,
16082                };
16083                #[cfg(any(test, feature = "benchmark_internal"))]
16084                {
16085                    let mut probe = self.recalc_reuse_probe.lock().unwrap();
16086                    probe.schedule_cache_hits += 1;
16087                    probe.schedule_shared_handles += 1;
16088                }
16089                return Ok((
16090                    EvaluationSchedule::Shared(Arc::clone(&cached.schedule)),
16091                    FxHashMap::default(),
16092                    meta,
16093                ));
16094            }
16095
16096            let (schedule, vdeps, mut meta) = match self.schedule_from_base(to_evaluate)? {
16097                Some(schedule) => (
16098                    schedule,
16099                    FxHashMap::default(),
16100                    ScheduleBuildMeta {
16101                        candidate_vertices: to_evaluate.len(),
16102                        vdeps_vertices: 0,
16103                        vdeps_edges: 0,
16104                        builder_elapsed_ms: 0,
16105                        used_virtual_schedule: false,
16106                        schedule_cache_hit: false,
16107                        schedule_cache_eligible: true,
16108                    },
16109                ),
16110                None => self.create_evaluation_schedule_active(to_evaluate)?,
16111            };
16112            meta.schedule_cache_hit = false;
16113            meta.schedule_cache_eligible = true;
16114            #[cfg(any(test, feature = "benchmark_internal"))]
16115            {
16116                self.recalc_reuse_probe
16117                    .get_mut()
16118                    .unwrap()
16119                    .schedule_cache_misses += 1;
16120            }
16121            let schedule = if vdeps.is_empty() {
16122                // Clone previously discarded builder spare capacity. Keep that compact
16123                // retained payload while sharing it with the current request.
16124                let mut schedule = schedule;
16125                schedule.units.shrink_to_fit();
16126                for layer in &mut schedule.layers {
16127                    layer.vertices.shrink_to_fit();
16128                }
16129                schedule.layers.shrink_to_fit();
16130                for cycle in &mut schedule.cycles {
16131                    cycle.shrink_to_fit();
16132                }
16133                schedule.cycles.shrink_to_fit();
16134                let schedule = Arc::new(schedule);
16135                #[cfg(any(test, feature = "benchmark_internal"))]
16136                {
16137                    self.recalc_reuse_probe
16138                        .get_mut()
16139                        .unwrap()
16140                        .schedule_shared_handles += 1;
16141                }
16142                let entry = CachedScheduleEntry {
16143                    topology_epoch: self.topology_epoch,
16144                    authority_revision: self.schedule_cache_authority_revision(),
16145                    candidate_vertices: VertexIdRuns::from_slice(to_evaluate),
16146                    schedule: Arc::clone(&schedule),
16147                };
16148                if let Some(previous) = self.cached_static_schedule.replace(entry) {
16149                    self.retain_recent_schedule(previous);
16150                }
16151                EvaluationSchedule::Shared(schedule)
16152            } else {
16153                EvaluationSchedule::Owned(schedule)
16154            };
16155            return Ok((schedule, vdeps, meta));
16156        }
16157
16158        let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_active(to_evaluate)?;
16159        meta.schedule_cache_hit = false;
16160        meta.schedule_cache_eligible = false;
16161        #[cfg(any(test, feature = "benchmark_internal"))]
16162        {
16163            self.recalc_reuse_probe
16164                .get_mut()
16165                .unwrap()
16166                .schedule_cache_ineligible += 1;
16167        }
16168        Ok((EvaluationSchedule::Owned(schedule), vdeps, meta))
16169    }
16170
16171    /// Plan reuse: the base schedule restricted to `to_evaluate` when it
16172    /// is current, covers the request, has at least
16173    /// `BASE_SCHEDULE_MIN_REQUEST` candidates, and is at most
16174    /// `BASE_SCHEDULE_RATIO` times its size (restricting walks the whole
16175    /// base; planning costs far more per candidate).
16176    fn schedule_from_base(
16177        &mut self,
16178        to_evaluate: &[VertexId],
16179    ) -> Result<Option<crate::engine::scheduler::Schedule>, ExcelError> {
16180        let revision = self.schedule_cache_authority_revision();
16181        let current = |e: &&CachedScheduleEntry| {
16182            e.topology_epoch == self.topology_epoch && e.authority_revision == revision
16183        };
16184        // The base, or the current schedule when larger (the first
16185        // evaluation's, before a later request replaces it).
16186        let Some(base) = [
16187            self.base_schedule.as_ref(),
16188            self.cached_static_schedule.as_ref(),
16189        ]
16190        .into_iter()
16191        .flatten()
16192        .filter(current)
16193        .max_by_key(|e| e.candidate_vertices.len()) else {
16194            return Ok(None);
16195        };
16196        let base_len = base.candidate_vertices.len();
16197        if base_len <= BASE_SCHEDULE_MIN_VERTICES
16198            || to_evaluate.len() < BASE_SCHEDULE_MIN_REQUEST
16199            || to_evaluate.len() > base_len
16200            || base_len > to_evaluate.len().saturating_mul(BASE_SCHEDULE_RATIO)
16201        {
16202            return Ok(None);
16203        }
16204        let mut keep = crate::engine::idset::DenseIdSet::default();
16205        keep.extend(to_evaluate.iter().copied());
16206        let Some((schedule, kept)) = base.schedule.restrict(&keep) else {
16207            return Ok(None);
16208        };
16209        // Every requested vertex must be in the base.
16210        if kept != keep.len() {
16211            return Ok(None);
16212        }
16213        if let Some(ledger) = self.active_resource_ledger.as_mut() {
16214            let layers: usize = schedule
16215                .layers
16216                .iter()
16217                .map(|l| {
16218                    l.vertices.capacity() * std::mem::size_of::<VertexId>()
16219                        + l.runs.capacity()
16220                            * std::mem::size_of::<crate::engine::scheduler::LayerRun>()
16221                })
16222                .sum();
16223            let bytes = (keep.heap_bytes()
16224                + layers
16225                + schedule.layers.capacity()
16226                    * std::mem::size_of::<crate::engine::scheduler::Layer>()
16227                + schedule.units.capacity()
16228                    * std::mem::size_of::<crate::engine::scheduler::ScheduleUnit>()
16229                + schedule
16230                    .cycles
16231                    .iter()
16232                    .map(|c| c.capacity() * 4)
16233                    .sum::<usize>()) as u64;
16234            ledger
16235                .reserve_schedule_discovery(bytes)
16236                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16237            ledger
16238                .release_scratch(bytes)
16239                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16240        }
16241        #[cfg(debug_assertions)]
16242        self.debug_check_restricted_schedule(to_evaluate, &schedule);
16243        #[cfg(any(test, feature = "benchmark_internal"))]
16244        {
16245            self.recalc_reuse_probe
16246                .get_mut()
16247                .unwrap()
16248                .schedule_base_restrictions += 1;
16249        }
16250        Ok(Some(schedule))
16251    }
16252
16253    /// Debug builds: a restricted schedule holds each requested vertex
16254    /// once and orders the request like a freshly planned schedule (every
16255    /// dependency among the request in an earlier unit).
16256    #[cfg(debug_assertions)]
16257    fn debug_check_restricted_schedule(
16258        &self,
16259        to_evaluate: &[VertexId],
16260        schedule: &crate::engine::scheduler::Schedule,
16261    ) {
16262        let mut position: FxHashMap<VertexId, usize> = FxHashMap::default();
16263        for (u, unit) in schedule.units.iter().enumerate() {
16264            let vertices: &[VertexId] = match *unit {
16265                crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16266                    &schedule.unit_layer(i).vertices
16267                }
16268                crate::engine::scheduler::ScheduleUnit::Cycle(i) => schedule.unit_cycle(i),
16269            };
16270            for &v in vertices {
16271                assert!(
16272                    position.insert(v, u).is_none(),
16273                    "vertex twice in a restricted schedule"
16274                );
16275            }
16276        }
16277        assert_eq!(position.len(), to_evaluate.len());
16278        if to_evaluate.len() > 512 {
16279            return;
16280        }
16281        // Every arc among the request (the store's edge images) goes to an
16282        // earlier unit, or within one sequential layer (a chain) or cycle.
16283        let Ok(store) = self.graph.authority_plan_store() else {
16284            return;
16285        };
16286        let cells: Vec<(VertexId, (u16, u32, u32))> = to_evaluate
16287            .iter()
16288            .filter_map(|&v| self.graph.authority_cell_of_vertex(v).map(|c| (v, c)))
16289            .collect();
16290        for &(v, (sheet, row, col)) in &cells {
16291            let Some(owner) = store.owner_at((sheet, row, col)) else {
16292                continue;
16293            };
16294            let Ok(refined) = store.refine_owner_column(owner, col, row, row, None) else {
16295                continue;
16296            };
16297            for piece in &refined.pieces {
16298                for edge in &refined.edges[piece.edge_start..piece.edge_end] {
16299                    let Some(image) = edge.proj.forward(&piece.domain) else {
16300                        continue;
16301                    };
16302                    for &(d, (s2, r2, c2)) in &cells {
16303                        if d == v
16304                            || edge.proj.sheet != s2
16305                            || !(image.r0 <= r2
16306                                && r2 <= image.r1
16307                                && image.c0 <= c2
16308                                && c2 <= image.c1)
16309                        {
16310                            continue;
16311                        }
16312                        let (pd, pv) = (position[&d], position[&v]);
16313                        let same_ok = pd == pv
16314                            && match schedule.units[pv] {
16315                                crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16316                                    schedule.unit_layer(i).sequential
16317                                }
16318                                crate::engine::scheduler::ScheduleUnit::Cycle(_) => true,
16319                            };
16320                        assert!(
16321                            pd < pv || same_ok,
16322                            "restricted schedule orders {v:?} before its precedent {d:?}"
16323                        );
16324                    }
16325                }
16326            }
16327        }
16328    }
16329
16330    /// Compress family formulas once per authority build (see
16331    /// `EvalConfig::formula_compression`), when no staged or deferred
16332    /// formula package can hold arena ids.
16333    fn maybe_compress_formulas(&mut self) {
16334        if !self.config.formula_compression {
16335            return;
16336        }
16337        let builds = self.graph.authority_host().builds;
16338        if self.compressed_at_build == Some(builds) {
16339            return;
16340        }
16341        self.compressed_at_build = Some(builds);
16342        if self.has_staged_formulas() {
16343            // Staged packages hold arena ids: no compaction. Members that
16344            // are already compressed can still leave the per-cell maps.
16345            self.graph.virtualize_family_members();
16346            return;
16347        }
16348        let pool = self.thread_pool.clone();
16349        let (_, garbage) = self.graph.compress_family_formulas(pool.as_deref());
16350        self.graph.virtualize_family_members();
16351        // Freeing the dropped members' reference texts (one allocation each)
16352        // is most of compaction; with a pool it happens off the critical
16353        // path.
16354        match pool {
16355            Some(pool) if garbage.len() >= 1024 => pool.spawn(move || drop(garbage)),
16356            _ => drop(garbage),
16357        }
16358    }
16359
16360    fn create_evaluation_schedule_active(
16361        &mut self,
16362        to_evaluate: &[VertexId],
16363    ) -> Result<ScheduleBuildOutput, ExcelError> {
16364        self.graph.authority_sync();
16365        self.maybe_compress_formulas();
16366        let mut ledger = self.active_resource_ledger.take();
16367        let result = self.create_evaluation_schedule_uncached(to_evaluate, ledger.as_mut());
16368        self.active_resource_ledger = ledger;
16369        result
16370    }
16371
16372    fn create_evaluation_schedule_uncached(
16373        &self,
16374        to_evaluate: &[VertexId],
16375        #[allow(unused_variables)] ledger: Option<&mut ResourceLedger>,
16376    ) -> Result<ScheduleBuildOutput, ExcelError> {
16377        #[cfg(any(test, feature = "benchmark_internal"))]
16378        {
16379            self.recalc_reuse_probe.lock().unwrap().schedule_builds += 1;
16380        }
16381        let builder = VirtualDepBuilder::new(self);
16382        #[allow(unused_mut)]
16383        let (mut vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
16384            if self.config.enable_virtual_dep_telemetry {
16385                let build_started = crate::instant::FzInstant::now();
16386                let (vdeps, augmented) = builder.build(to_evaluate);
16387                let builder_elapsed_ms = build_started.elapsed().as_millis();
16388                let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
16389                (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
16390            } else {
16391                let (vdeps, augmented) = builder.build(to_evaluate);
16392                (vdeps, augmented, 0, 0)
16393            };
16394
16395        // Replan hints from stale dynamic reads earlier in this request.
16396        {
16397            self.freshness_merge_hints(to_evaluate, &mut vdeps);
16398            self.freshness_extent_hints(to_evaluate, &mut vdeps);
16399        }
16400        let mut final_evaluate = to_evaluate.to_vec();
16401        if !augmented.is_empty() {
16402            final_evaluate.extend(augmented);
16403            final_evaluate.sort_unstable();
16404            final_evaluate.dedup();
16405        }
16406
16407        let use_virtual = !vdeps.is_empty();
16408
16409        let schedule = self.create_authority_schedule(&final_evaluate, &vdeps, ledger)?;
16410
16411        let meta = ScheduleBuildMeta {
16412            candidate_vertices: to_evaluate.len(),
16413            vdeps_vertices: vdeps.len(),
16414            vdeps_edges,
16415            builder_elapsed_ms,
16416            used_virtual_schedule: use_virtual,
16417            schedule_cache_hit: false,
16418            schedule_cache_eligible: false,
16419        };
16420
16421        Ok((schedule, vdeps, meta))
16422    }
16423
16424    fn create_authority_schedule(
16425        &self,
16426        candidates: &[VertexId],
16427        vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16428        mut ledger: Option<&mut ResourceLedger>,
16429    ) -> Result<crate::engine::scheduler::Schedule, ExcelError> {
16430        use crate::engine::authority::{
16431            geom::{Cover, Rect},
16432            plan_schedule, planner,
16433            proj::{AxisMap, RefProj},
16434            store::{EdgeKey, Tag},
16435        };
16436        let failure = |message: String| {
16437            ExcelError::new(ExcelErrorKind::Error)
16438                .with_message(format!("unified_authority planner: {message}"))
16439        };
16440        self.cancellation_checkpoint("Evaluation cancelled before authority planning")?;
16441        let store = self
16442            .graph
16443            .authority_plan_store()
16444            .map_err(Self::authority_excel_error)?;
16445        // One formula cell without hints (a tiny edit): its plan is the cell
16446        // alone unless it reads itself (`planner::plan_single`).
16447        if let [only] = candidates
16448            && vdeps.is_empty()
16449            && self.graph.authority_host().observed(*only).is_none()
16450            && let Some(cell) = self.graph.authority_cell_of_vertex(*only)
16451            && cell.0 != crate::engine::authority::geom::SYMBOL_SHEET
16452            && let Some(single) = planner::plan_single(store, cell)
16453        {
16454            #[cfg(debug_assertions)]
16455            {
16456                let mut cover = Cover::new();
16457                cover.insert_rect(cell.0, &Rect::new(cell.1, cell.2, cell.1, cell.2));
16458                let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16459                    .expect("general plan of one cell");
16460                assert_eq!(
16461                    general.cells.as_slice(),
16462                    &[single],
16463                    "single-cell plan differs from the planner at {cell:?}"
16464                );
16465            }
16466            let adapted = plan_schedule::schedule(
16467                &[single],
16468                0,
16469                None,
16470                |cell| {
16471                    self.graph
16472                        .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16473                        .ok_or_else(|| failure("missing executor identity".to_owned()))
16474                },
16475                |_work| Ok(()),
16476            )
16477            .map_err(|error| match error {
16478                plan_schedule::ScheduleError::Runtime(error) => error,
16479                other => failure(format!("{other:?}")),
16480            })?;
16481            if let Some(ledger) = ledger {
16482                ledger
16483                    .reserve_schedule_discovery(adapted.peak_heap_bytes)
16484                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16485                ledger
16486                    .release_scratch(adapted.peak_heap_bytes)
16487                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16488            }
16489            return Ok(adapted.schedule);
16490        }
16491        // A small request of grid formula cells without hints: per-cell
16492        // arcs and longest-path levels (`planner::plan_small`).
16493        if (2..=planner::SMALL_PLAN_MAX).contains(&candidates.len())
16494            && vdeps.is_empty()
16495            && candidates
16496                .iter()
16497                .all(|&v| self.graph.authority_host().observed(v).is_none())
16498        {
16499            let cells: Option<Vec<(u16, u32, u32)>> = candidates
16500                .iter()
16501                .map(|&v| self.graph.authority_cell_of_vertex(v))
16502                .collect();
16503            if let Some(small) = cells.as_deref().and_then(|c| planner::plan_small(store, c)) {
16504                #[cfg(debug_assertions)]
16505                {
16506                    let mut cover = Cover::new();
16507                    for c in cells.as_deref().unwrap_or_default() {
16508                        cover.insert_rect(c.0, &Rect::new(c.1, c.2, c.1, c.2));
16509                    }
16510                    let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16511                        .expect("general plan of a small request");
16512                    let key = |c: &planner::OrderedCell| (c.sheet, c.row, c.col, c.id, c.owner);
16513                    let mut a: Vec<_> = general.cells.iter().map(key).collect();
16514                    let mut b: Vec<_> = small.iter().map(key).collect();
16515                    a.sort_unstable();
16516                    b.sort_unstable();
16517                    assert_eq!(a, b, "small plan cells differ from the planner");
16518                    assert!(
16519                        general.cells.iter().all(|c| c.cycle.is_none()),
16520                        "small plan of a cyclic request"
16521                    );
16522                }
16523                let adapted = plan_schedule::schedule(
16524                    &small,
16525                    0,
16526                    None,
16527                    |cell| {
16528                        self.graph
16529                            .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16530                            .ok_or_else(|| failure("missing executor identity".to_owned()))
16531                    },
16532                    |_work| Ok(()),
16533                )
16534                .map_err(|error| match error {
16535                    plan_schedule::ScheduleError::Runtime(error) => error,
16536                    other => failure(format!("{other:?}")),
16537                })?;
16538                if let Some(ledger) = ledger {
16539                    ledger
16540                        .reserve_schedule_discovery(adapted.peak_heap_bytes)
16541                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16542                    ledger
16543                        .release_scratch(adapted.peak_heap_bytes)
16544                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16545                }
16546                return Ok(adapted.schedule);
16547            }
16548        }
16549        // Names are symbol-plane nodes (design §4.1): a name vertex plans as
16550        // the unit at its node, between its precedents and its readers.
16551        // Candidates become cells, sorted by (sheet, column, row) and
16552        // coalesced into row intervals: one cover insert per interval.
16553        let mut cover = Cover::new();
16554        {
16555            let cell_of = |&id: &VertexId| {
16556                self.graph
16557                    .authority_cell_of_vertex(id)
16558                    .map(|(sheet, row, col)| (sheet, col, row))
16559            };
16560            // A full recalc maps and sorts every formula: on the pool when
16561            // there is one (first eval's schedule is serial work otherwise).
16562            let cells: Vec<(u16, u32, u32)> = match self.thread_pool.as_deref() {
16563                Some(pool) if candidates.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16564                    use rayon::prelude::*;
16565                    pool.install(|| {
16566                        let mut cells: Vec<_> = candidates.par_iter().filter_map(cell_of).collect();
16567                        cells.par_sort_unstable();
16568                        cells
16569                    })
16570                }
16571                _ => {
16572                    let mut cells: Vec<_> = candidates.iter().filter_map(cell_of).collect();
16573                    cells.sort_unstable();
16574                    cells
16575                }
16576            };
16577            let mut i = 0;
16578            while i < cells.len() {
16579                let (sheet, col, r0) = cells[i];
16580                let mut r1 = r0;
16581                let mut j = i + 1;
16582                while j < cells.len() && cells[j].0 == sheet && cells[j].1 == col {
16583                    if cells[j].2 > r1 + 1 {
16584                        break;
16585                    }
16586                    r1 = r1.max(cells[j].2);
16587                    j += 1;
16588                }
16589                cover.insert_rect(sheet, &Rect::new(r0, col, r1, col));
16590                i = j;
16591            }
16592        }
16593        let mut hints = Vec::new();
16594        for (&reader, deps) in vdeps {
16595            let Some(reader) = self.graph.authority_cell_of_vertex(reader) else {
16596                continue;
16597            };
16598            for &dependency in deps {
16599                let Some(dep) = self.graph.authority_cell_of_vertex(dependency) else {
16600                    continue;
16601                };
16602                hints.push(planner::PlanHint {
16603                    reader: (reader.0, reader.2, reader.1),
16604                    edge: EdgeKey {
16605                        dep_sheet: reader.0,
16606                        tag: Tag::X,
16607                        lk: u32::MAX,
16608                        proj: RefProj {
16609                            sheet: dep.0,
16610                            rows: AxisMap::fixed(dep.1, dep.1),
16611                            cols: AxisMap::fixed(dep.2, dep.2),
16612                        },
16613                    },
16614                });
16615            }
16616        }
16617        // rdi_dyn: order each dynamic reader after its observed reads.
16618        let host = self.graph.authority_host();
16619        for &id in candidates.iter().filter(|_| host.has_observed()) {
16620            let Some(reads) = host.observed(id) else {
16621                continue;
16622            };
16623            let Some(reader) = self.graph.authority_cell_of_vertex(id) else {
16624                continue;
16625            };
16626            for &(sheet, r0, c0, r1, c1) in reads {
16627                hints.push(planner::PlanHint {
16628                    reader: (reader.0, reader.2, reader.1),
16629                    edge: EdgeKey {
16630                        dep_sheet: reader.0,
16631                        tag: Tag::X,
16632                        lk: u32::MAX,
16633                        proj: RefProj {
16634                            sheet,
16635                            rows: AxisMap::fixed(r0, r1),
16636                            cols: AxisMap::fixed(c0, c1),
16637                        },
16638                    },
16639                });
16640            }
16641        }
16642        hints.sort_unstable_by_key(|hint| hint.reader);
16643        let checkpoint = ledger
16644            .as_ref()
16645            .map_or(0, |ledger| ledger.scratch_checkpoint());
16646        let scratch_limit = ledger
16647            .as_ref()
16648            .and_then(|ledger| ledger.schedule_discovery_limit())
16649            .map(|limit| limit.saturating_sub(checkpoint));
16650        let ordered = planner::plan_with_hints(store, &cover, &hints, scratch_limit, None, None)
16651            .map_err(|error| failure(format!("{error:?}")))?;
16652        // max_work_units is an execution budget. Planning work must be capped
16653        // independently: charging it here changes the observable publication
16654        // boundary (e.g. a spill must commit before the next execution fails).
16655        let adapted = plan_schedule::schedule(
16656            &ordered.cells,
16657            ordered.heap_bytes(),
16658            scratch_limit,
16659            |cell| {
16660                self.graph
16661                    .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16662                    .ok_or_else(|| failure("missing executor identity".to_owned()))
16663            },
16664            |_work| {
16665                self.cancellation_checkpoint("Evaluation cancelled during authority planning")?;
16666                if let Some(ledger) = ledger.as_deref_mut() {
16667                    ledger
16668                        .checkpoint_deadline()
16669                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16670                }
16671                Ok(())
16672            },
16673        )
16674        .map_err(|error| match error {
16675            plan_schedule::ScheduleError::Runtime(error) => error,
16676            other => failure(format!("{other:?}")),
16677        })?;
16678        if let Some(ledger) = ledger {
16679            let peak = ordered.peak_heap_bytes.max(adapted.peak_heap_bytes);
16680            ledger
16681                .reserve_schedule_discovery(peak)
16682                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16683            ledger
16684                .release_scratch(peak)
16685                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16686        }
16687        // The planner's order is scratch now; a full recalc's is large.
16688        match self.thread_pool.as_deref() {
16689            Some(pool) if ordered.cells.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16690                pool.spawn(move || drop(ordered))
16691            }
16692            _ => drop(ordered),
16693        }
16694        Ok(adapted.schedule)
16695    }
16696
16697    /// Static-schedule cache eligibility. Legacy excludes range readers:
16698    /// their order comes from per-request range virtual deps. Under the
16699    /// authority a range read is a static edge of the relation, so only
16700    /// dynamic readers (whose hints are per request) are excluded, and the
16701    /// key adds the authority revision (design §8.4).
16702    fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
16703        {
16704            // A dynamic reader is planned from its observed reads, which the
16705            // key covers (rev.dyn); one without them needs a pre-probe, and
16706            // request-scoped replan hints are never cached.
16707            let host = self.graph.authority_host();
16708            !to_evaluate.is_empty()
16709                && !self.freshness_has_hints()
16710                && to_evaluate
16711                    .iter()
16712                    .all(|&v| !self.graph.is_dynamic(v) || host.observed(v).is_some())
16713        }
16714    }
16715
16716    fn schedule_cache_authority_revision(&self) -> (u64, u64) {
16717        {
16718            // rev.topology is `topology_epoch`; a symbol revision rebuilds the
16719            // store, so it is part of `revision`.
16720            let host = self.graph.authority_host();
16721            (host.revision(), host.rev_dyn())
16722        }
16723    }
16724
16725    fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
16726        VirtualDepTelemetry {
16727            fallback_mode_activations: self.virtual_dep_fallback_activations,
16728            ..VirtualDepTelemetry::default()
16729        }
16730    }
16731
16732    fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
16733        telemetry.candidate_vertices_total += meta.candidate_vertices;
16734        telemetry.vdeps_vertices_total += meta.vdeps_vertices;
16735        telemetry.vdeps_edges_total += meta.vdeps_edges;
16736        telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
16737        if meta.schedule_cache_eligible {
16738            if meta.schedule_cache_hit {
16739                telemetry.schedule_cache_hits += 1;
16740                telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
16741            } else {
16742                telemetry.schedule_cache_misses += 1;
16743            }
16744        }
16745        if meta.used_virtual_schedule {
16746            telemetry.schedule_virtual_passes += 1;
16747        } else {
16748            telemetry.schedule_static_passes += 1;
16749        }
16750    }
16751
16752    /// End-of-pass dirty bookkeeping; true when the loop must replan.
16753    /// Legacy: clear the pass, re-dirty readers whose pre-probe changed.
16754    /// Under the authority an armed pass keeps stale and unreached vertices
16755    /// dirty instead (design §8.2, `freshness.rs`).
16756    fn finish_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16757        self.finish_pass_dirty_scoped(to_evaluate, changed, true)
16758    }
16759
16760    /// [`Self::finish_pass_dirty`] for a targeted pass: only its candidates
16761    /// decide whether to replan.
16762    fn finish_target_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16763        self.finish_pass_dirty_scoped(to_evaluate, changed, false)
16764    }
16765
16766    fn finish_pass_dirty_scoped(
16767        &mut self,
16768        to_evaluate: &[VertexId],
16769        changed: &[VertexId],
16770        #[allow(unused_variables)] whole_workbook: bool,
16771    ) -> bool {
16772        self.freshness_finish_pass(to_evaluate, changed, whole_workbook)
16773    }
16774
16775    /// Start a pass over `schedule` (arms the freshness recorder).
16776    fn begin_pass(
16777        &mut self,
16778        #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16779    ) {
16780        self.freshness_begin_pass(schedule);
16781    }
16782
16783    /// FR4 layer barrier after unit `index`: true stops the pass.
16784    fn stop_after_unit(
16785        &mut self,
16786        #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16787        #[allow(unused_variables)] index: usize,
16788    ) -> bool {
16789        self.freshness_stop_after_unit(schedule, index)
16790    }
16791
16792    fn changed_virtual_dep_vertices(
16793        &mut self,
16794        to_evaluate: &[VertexId],
16795        old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16796    ) -> Vec<VertexId> {
16797        #[cfg(test)]
16798        if self.force_virtual_dep_changes_remaining_for_test > 0
16799            && let Some(vertex) = to_evaluate.first().copied()
16800        {
16801            self.force_virtual_dep_changes_remaining_for_test -= 1;
16802            return vec![vertex];
16803        }
16804        // An armed pass detects stale dynamic reads directly; the pre-probe
16805        // comparison is legacy's substitute for that (design §8.2).
16806        if self.freshness_armed() {
16807            return Vec::new();
16808        }
16809        if !to_evaluate
16810            .iter()
16811            .copied()
16812            .any(|v| self.graph.is_dynamic(v))
16813        {
16814            return Vec::new();
16815        }
16816
16817        let builder = VirtualDepBuilder::new(self);
16818        let (new_vdeps, _) = builder.build(to_evaluate);
16819
16820        let mut candidates = FxHashSet::default();
16821        candidates.extend(old_vdeps.keys().copied());
16822        candidates.extend(new_vdeps.keys().copied());
16823
16824        let mut changed = Vec::new();
16825        for v in candidates {
16826            if old_vdeps.get(&v) != new_vdeps.get(&v) {
16827                changed.push(v);
16828            }
16829        }
16830        changed
16831    }
16832
16833    /// Build a demand-driven subgraph for the given targets, including ephemeral edges for
16834    /// compressed ranges, and returning the set of dirty/volatile precedents and virtual deps.
16835    /// Demand candidates of `targets` and their dynamic plan hints: under
16836    /// `unified_authority` from the authority's relation (design §8.3),
16837    /// otherwise from the legacy graph.
16838    #[allow(clippy::type_complexity)]
16839    fn demand_subgraph(
16840        &self,
16841        targets: &[VertexId],
16842    ) -> Result<
16843        (
16844            Vec<VertexId>,
16845            rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16846        ),
16847        ExcelError,
16848    > {
16849        self.authority_demand_subgraph(targets)
16850    }
16851
16852    /// Design §8.3: traverse precedents from the targets over the
16853    /// authority's static relation (symbol nodes are ordinary pieces) plus
16854    /// the dynamic readers' virtual dependencies, and collect what legacy's
16855    /// demand walk collects: dirty or volatile formula cells and every name
16856    /// passed through. No legacy dependency structure is read.
16857    #[allow(clippy::type_complexity)]
16858    fn authority_demand_subgraph(
16859        &self,
16860        targets: &[VertexId],
16861    ) -> Result<
16862        (
16863            Vec<VertexId>,
16864            rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16865        ),
16866        ExcelError,
16867    > {
16868        use crate::engine::authority::geom::{Cell, Rect, SYMBOL_SHEET};
16869        use crate::engine::authority::store::TagFilter;
16870        use rustc_hash::{FxHashMap, FxHashSet};
16871        let store = self
16872            .graph
16873            .authority_plan_store()
16874            .map_err(Self::authority_excel_error)?;
16875        let ids = store.ids();
16876        let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
16877        let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default();
16878        let mut visited: FxHashSet<Cell> = FxHashSet::default();
16879        // (cell, authority id or NO_VID): the id lets the side array
16880        // translate the cell without a hash lookup.
16881        let mut stack: Vec<(Cell, u32)> = Vec::new();
16882        // Formula cells under `rect` on `sheet`: identity runs per column.
16883        let push_formulas = |stack: &mut Vec<(Cell, u32)>,
16884                             visited: &FxHashSet<Cell>,
16885                             sheet: u16,
16886                             rect: Rect| {
16887            for col in rect.c0..=rect.c1 {
16888                ids.visit_runs_in(sheet, col, rect.r0, rect.r1, &mut |h| {
16889                    let run = ids.run(h);
16890                    let r0 = run.row_start.max(rect.r0);
16891                    let r1 = (run.row_start + run.len - 1).min(rect.r1);
16892                    for row in r0..=r1 {
16893                        if !visited.contains(&(sheet, row, col)) {
16894                            stack.push(((sheet, row, col), run.first_id + (row - run.row_start)));
16895                        }
16896                    }
16897                });
16898            }
16899        };
16900        for &v in targets {
16901            if let Some(table) = self.graph.table_by_vertex(v) {
16902                // A table's demand is its range's, as legacy's table vertex
16903                // leads to the cells it covers (its symbol row has no
16904                // precedents).
16905                let (s, e) = (table.range.start, table.range.end);
16906                push_formulas(
16907                    &mut stack,
16908                    &visited,
16909                    s.sheet_id,
16910                    Rect::new(s.coord.row(), s.coord.col(), e.coord.row(), e.coord.col()),
16911                );
16912            } else if let Some(cell) = self.graph.authority_cell_of_vertex(v) {
16913                stack.push((cell, crate::engine::authority::identity::NO_VID));
16914            }
16915        }
16916        let mut hits = Vec::new();
16917        #[cfg(any(test, feature = "benchmark_internal"))]
16918        let (mut probe_vertices, mut probe_clean_formulas, mut probe_edges, mut probe_dynamic) =
16919            (0, 0, 0, 0);
16920        while let Some((cell, id)) = stack.pop() {
16921            if !visited.insert(cell) {
16922                continue;
16923            }
16924            let Some(v) = self.graph.authority_vertex_of_formula(id, cell) else {
16925                continue;
16926            };
16927            if !self.graph.vertex_exists(v) {
16928                continue;
16929            }
16930            #[cfg(any(test, feature = "benchmark_internal"))]
16931            {
16932                probe_vertices += 1;
16933            }
16934            match self.graph.get_vertex_kind(v) {
16935                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
16936                    if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
16937                        to_evaluate.insert(v);
16938                    } else {
16939                        #[cfg(any(test, feature = "benchmark_internal"))]
16940                        {
16941                            probe_clean_formulas += 1;
16942                        }
16943                    }
16944                }
16945                VertexKind::NamedScalar | VertexKind::NamedArray => {
16946                    to_evaluate.insert(v);
16947                }
16948                _ => {}
16949            }
16950            hits.clear();
16951            store.direct_precedents(cell, TagFilter::All, &mut hits);
16952            #[cfg(any(test, feature = "benchmark_internal"))]
16953            {
16954                probe_edges += hits.len();
16955            }
16956            for &(_, sheet, rect) in &hits {
16957                // DirtyExtents: a dirty spill anchor whose extent meets this
16958                // image is a demand precedent, ordered before `v` (§8.2).
16959                if sheet != SYMBOL_SHEET {
16960                    for anchor in self
16961                        .graph
16962                        .spill_anchors_in_region(sheet, rect.r0, rect.c0, rect.r1, rect.c1)
16963                    {
16964                        if anchor != v && self.graph.is_dirty(anchor) {
16965                            vdeps.entry(v).or_default().push(anchor);
16966                            if let Some(c) = self.graph.authority_cell_of_vertex(anchor) {
16967                                stack.push((c, crate::engine::authority::identity::NO_VID));
16968                            }
16969                        }
16970                    }
16971                }
16972                if sheet == SYMBOL_SHEET {
16973                    stack.extend((rect.r0..=rect.r1).map(|slot| {
16974                        (
16975                            (SYMBOL_SHEET, slot, 0),
16976                            crate::engine::authority::identity::NO_VID,
16977                        )
16978                    }));
16979                    continue;
16980                }
16981                push_formulas(&mut stack, &visited, sheet, rect);
16982            }
16983            if self.graph.is_dynamic(v) {
16984                #[cfg(any(test, feature = "benchmark_internal"))]
16985                {
16986                    probe_dynamic += 1;
16987                }
16988                // rdi_dyn: the observed reads are demand precedents.
16989                if let Some(reads) = self.graph.authority_host().observed(v) {
16990                    for &(sheet, r0, c0, r1, c1) in reads {
16991                        push_formulas(&mut stack, &visited, sheet, Rect::new(r0, c0, r1, c1));
16992                    }
16993                }
16994                let (vdeps_map, _) = VirtualDepBuilder::new(self).build(&[v]);
16995                // Pre-probe targets plus reads this request found dirty
16996                // (design §8.3: demand walks rdi ∪ hints).
16997                let hinted = self.freshness_hints(v).unwrap_or(&[]);
16998                if let Some(deps) = vdeps_map
16999                    .get(&v)
17000                    .map(|deps| deps.iter().chain(hinted))
17001                    .or(Some([].iter().chain(hinted)))
17002                {
17003                    for &u in deps {
17004                        vdeps.entry(v).or_default().push(u);
17005                        if let Some(c) = self.graph.authority_cell_of_vertex(u) {
17006                            stack.push((c, crate::engine::authority::identity::NO_VID));
17007                        }
17008                    }
17009                }
17010            }
17011        }
17012        let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
17013        result.sort_unstable();
17014        for deps in vdeps.values_mut() {
17015            deps.sort_unstable();
17016            deps.dedup();
17017        }
17018        #[cfg(any(test, feature = "benchmark_internal"))]
17019        {
17020            let mut probe = self.recalc_reuse_probe.lock().unwrap();
17021            probe.demand_builds += 1;
17022            probe.demand_vertices += probe_vertices;
17023            probe.demand_clean_formulas += probe_clean_formulas;
17024            probe.demand_explicit_edges += probe_edges;
17025            probe.demand_virtual_builder_calls += probe_dynamic;
17026        }
17027        Ok((result, vdeps))
17028    }
17029
17030    /// Helper: convert 1-based column index to Excel-style letters (1 -> A, 27 -> AA)
17031    fn col_to_letters(col: u32) -> String {
17032        col_letters_from_1based(col).expect("column index must be >= 1")
17033    }
17034
17035    /// Evaluate all dirty/volatile vertices with cancellation support
17036    pub fn evaluate_all_cancellable(
17037        &mut self,
17038        cancel: crate::engine::CancelToken,
17039    ) -> Result<EvalResult, ExcelError> {
17040        self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
17041            engine.observe_function_semantic_epoch()?;
17042            engine.active_cancel_flag = Some(cancel.clone());
17043            let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
17044            engine.active_cancel_flag = None;
17045            res
17046        })
17047    }
17048
17049    fn evaluate_all_cancellable_impl(
17050        &mut self,
17051        cancel_flag: &AtomicBool,
17052    ) -> Result<EvalResult, ExcelError> {
17053        let _source_cache = self.source_cache_session();
17054        self.validate_deterministic_mode()?;
17055        if self.config.defer_graph_building {
17056            self.build_graph_all()?;
17057        }
17058        if cancel_flag.load(Ordering::Relaxed) {
17059            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17060                .with_message("Evaluation cancelled before scheduling".to_string()));
17061        }
17062        self.require_unified_authority()?;
17063        self.begin_evaluation_request();
17064        self.reset_virtual_dep_telemetry_if_disabled();
17065        let start = crate::instant::FzInstant::now();
17066        let mut computed_vertices = 0;
17067        let mut cycle_errors = 0;
17068
17069        let mut replan_iterations = 0;
17070        const MAX_REPLAN: usize = 5;
17071        let mut telemetry = self
17072            .config
17073            .enable_virtual_dep_telemetry
17074            .then(|| self.start_virtual_dep_telemetry());
17075
17076        loop {
17077            if cancel_flag.load(Ordering::Relaxed) {
17078                if let Some(mut t) = telemetry {
17079                    t.bailout_reason = Some("cancelled");
17080                    t.replan_iterations = replan_iterations;
17081                    self.last_virtual_dep_telemetry = t;
17082                }
17083                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17084                    .with_message("Evaluation cancelled before scheduling".to_string()));
17085            }
17086
17087            let to_evaluate = self.graph.get_evaluation_vertices();
17088            if to_evaluate.is_empty() {
17089                if let Some(t) = telemetry.as_mut()
17090                    && t.bailout_reason.is_none()
17091                {
17092                    t.bailout_reason = Some("no_work");
17093                }
17094                break;
17095            }
17096
17097            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
17098            if let Some(t) = telemetry.as_mut() {
17099                Self::accumulate_schedule_meta(t, &meta);
17100            }
17101
17102            // Walk units in condensation order, checking cancellation between
17103            // units (formerly between cycles and between layers).
17104            self.begin_pass(&schedule);
17105            for (unit_index, &unit) in schedule.units.iter().enumerate() {
17106                match unit {
17107                    ScheduleUnit::Cycle(i) => {
17108                        // Check cancellation between cycles
17109                        if cancel_flag.load(Ordering::Relaxed) {
17110                            if let Some(mut t) = telemetry {
17111                                t.bailout_reason = Some("cancelled");
17112                                t.replan_iterations = replan_iterations;
17113                                self.last_virtual_dep_telemetry = t;
17114                            }
17115                            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
17116                                "Evaluation cancelled during cycle handling".to_string(),
17117                            ));
17118                        }
17119
17120                        if self.handle_cycle_unit(
17121                            schedule.unit_cycle(i),
17122                            None,
17123                            None,
17124                            Some(cancel_flag),
17125                        )? > 0
17126                        {
17127                            cycle_errors += 1;
17128                        }
17129                    }
17130                    ScheduleUnit::Layer(i) => {
17131                        let layer = schedule.unit_layer(i);
17132                        // Check cancellation between layers
17133                        if cancel_flag.load(Ordering::Relaxed) {
17134                            if let Some(mut t) = telemetry {
17135                                t.bailout_reason = Some("cancelled");
17136                                t.replan_iterations = replan_iterations;
17137                                self.last_virtual_dep_telemetry = t;
17138                            }
17139                            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17140                                .with_message("Evaluation cancelled between layers".to_string()));
17141                        }
17142
17143                        // Evaluate vertices in this layer (parallel or sequential)
17144                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
17145                            computed_vertices +=
17146                                self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
17147                        } else {
17148                            computed_vertices +=
17149                                self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
17150                        }
17151                    }
17152                }
17153                if self.stop_after_unit(&schedule, unit_index) {
17154                    break;
17155                }
17156            }
17157
17158            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
17159            if let Some(t) = telemetry.as_mut() {
17160                t.changed_vdeps_total += changed_vertices.len();
17161            }
17162            self.resource_checkpoint(0)?;
17163            if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
17164                if let Some(t) = telemetry.as_mut() {
17165                    t.bailout_reason = Some("converged");
17166                }
17167                break;
17168            }
17169            if replan_iterations >= MAX_REPLAN {
17170                if let Some(mut t) = telemetry.take() {
17171                    t.bailout_reason = Some("max_replan");
17172                    t.replan_iterations = replan_iterations;
17173                    self.last_virtual_dep_telemetry = t;
17174                }
17175                return Err(
17176                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
17177                );
17178            }
17179            replan_iterations += 1;
17180        }
17181
17182        if let Some(mut t) = telemetry {
17183            t.replan_iterations = replan_iterations;
17184            self.last_virtual_dep_telemetry = t;
17185        }
17186
17187        // Re-dirty volatile vertices for the next evaluation cycle
17188        self.redirty_for_next_recalc();
17189        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
17190
17191        Ok(EvalResult {
17192            computed_vertices,
17193            cycle_errors,
17194            elapsed: start.elapsed(),
17195        })
17196    }
17197
17198    /// Evaluate only the necessary precedents for specific target cells with cancellation support
17199    pub fn evaluate_until_cancellable(
17200        &mut self,
17201        targets: &[&str],
17202        cancel: crate::engine::CancelToken,
17203    ) -> Result<EvalResult, ExcelError> {
17204        self.observe_evaluation_resource_request(
17205            EvaluationRequestKind::TargetedCancellable,
17206            |engine| {
17207                engine.observe_function_semantic_epoch()?;
17208                engine.active_cancel_flag = Some(cancel.clone());
17209                let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
17210                engine.active_cancel_flag = None;
17211                res
17212            },
17213        )
17214    }
17215
17216    fn evaluate_until_cancellable_impl(
17217        &mut self,
17218        targets: &[&str],
17219        cancel_flag: &AtomicBool,
17220    ) -> Result<EvalResult, ExcelError> {
17221        if cancel_flag.load(Ordering::Relaxed) {
17222            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17223                .with_message("Evaluation cancelled before target preparation"));
17224        }
17225        let mut typed_targets = Vec::with_capacity(targets.len());
17226        for target in targets {
17227            let (sheet, row, col) = self.parse_a1_notation(target)?;
17228            self.graph.sheet_id_mut(&sheet);
17229            typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
17230        }
17231        self.evaluate_mixed_targets(&typed_targets, None)
17232    }
17233
17234    fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
17235        let mut quoted = false;
17236        let mut separator = None;
17237        let bytes = address.as_bytes();
17238        let mut index = 0usize;
17239        while index < bytes.len() {
17240            match bytes[index] {
17241                b'\'' => {
17242                    if quoted && bytes.get(index + 1) == Some(&b'\'') {
17243                        index = index.saturating_add(1);
17244                    } else {
17245                        quoted = !quoted;
17246                    }
17247                }
17248                b'!' if !quoted => separator = Some(index),
17249                _ => {}
17250            }
17251            index = index.saturating_add(1);
17252        }
17253        if quoted {
17254            return Err(ExcelError::new(ExcelErrorKind::Ref)
17255                .with_message(format!("Invalid quoted sheet reference `{address}`")));
17256        }
17257        let (sheet, cell_part) = match separator {
17258            Some(separator) => {
17259                let raw_sheet = &address[..separator];
17260                let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
17261                    raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
17262                } else {
17263                    raw_sheet.to_string()
17264                };
17265                (sheet, &address[separator + 1..])
17266            }
17267            None => (self.default_sheet_name().to_string(), address),
17268        };
17269
17270        let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
17271            ExcelError::new(ExcelErrorKind::Ref)
17272                .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
17273        })?;
17274
17275        Ok((sheet, row, col))
17276    }
17277
17278    /// Determine volatility using this engine's FunctionProvider, falling back to global registry.
17279    fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
17280        use formualizer_parse::parser::ASTNodeType;
17281        match &ast.node_type {
17282            ASTNodeType::Function { name, args, .. } => {
17283                if let Some(func) = self
17284                    .get_function("", name)
17285                    .or_else(|| crate::function_registry::get("", name))
17286                    && func.caps().contains(crate::function::FnCaps::VOLATILE)
17287                {
17288                    return true;
17289                }
17290                args.iter()
17291                    .any(|arg| self.is_ast_volatile_with_provider(arg))
17292            }
17293            ASTNodeType::BinaryOp { left, right, .. } => {
17294                self.is_ast_volatile_with_provider(left)
17295                    || self.is_ast_volatile_with_provider(right)
17296            }
17297            ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
17298            ASTNodeType::Array(rows) => rows.iter().any(|row| {
17299                row.iter()
17300                    .any(|cell| self.is_ast_volatile_with_provider(cell))
17301            }),
17302            _ => false,
17303        }
17304    }
17305
17306    /// Evaluate a layer sequentially
17307    fn evaluate_layer_sequential(
17308        &mut self,
17309        layer: &super::scheduler::Layer,
17310    ) -> Result<usize, ExcelError> {
17311        self.resource_checkpoint(layer.vertices.len() as u64)?;
17312        self.evaluate_layer_sequential_effects(layer)
17313    }
17314
17315    fn update_vertex_value_with_delta(
17316        &mut self,
17317        vertex_id: VertexId,
17318        new_value: LiteralValue,
17319        delta: &mut DeltaCollector,
17320    ) {
17321        if delta.mode != DeltaMode::Off
17322            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
17323        {
17324            let sheet_name = self.graph.sheet_name(cell.sheet_id);
17325            let old = self
17326                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
17327                .unwrap_or(LiteralValue::Empty);
17328            if old != new_value {
17329                delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17330            }
17331        }
17332        self.graph.update_vertex_value_ref(vertex_id, &new_value);
17333        self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
17334    }
17335
17336    fn evaluate_layer_sequential_with_delta(
17337        &mut self,
17338        layer: &super::scheduler::Layer,
17339        delta: &mut DeltaCollector,
17340    ) -> Result<usize, ExcelError> {
17341        self.resource_checkpoint(layer.vertices.len() as u64)?;
17342        self.evaluate_layer_sequential_with_delta_effects(layer, delta)
17343    }
17344
17345    /// Evaluate a layer sequentially with cancellation support
17346    fn evaluate_layer_sequential_cancellable(
17347        &mut self,
17348        layer: &super::scheduler::Layer,
17349        cancel_flag: &AtomicBool,
17350    ) -> Result<usize, ExcelError> {
17351        self.resource_checkpoint(layer.vertices.len() as u64)?;
17352        self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
17353    }
17354
17355    /// Evaluate a layer sequentially with more frequent cancellation checks for demand-driven evaluation
17356    fn evaluate_layer_sequential_cancellable_demand_driven(
17357        &mut self,
17358        layer: &super::scheduler::Layer,
17359        cancel_flag: &AtomicBool,
17360    ) -> Result<usize, ExcelError> {
17361        self.resource_checkpoint(layer.vertices.len() as u64)?;
17362        self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
17363    }
17364
17365    /// Evaluate a layer in parallel using the thread pool.
17366    ///
17367    /// Cost-adaptive: the layer starts sequentially in slices of doubling
17368    /// size (1, 2, 4, ... vertices, capped so a slice does not overshoot the
17369    /// probe) and hands the rest to the pool once the rest looks worth it
17370    /// (`PARALLEL_LAYER_WORTH` at the rate so far) or the probe
17371    /// (`PARALLEL_LAYER_PROBE`) is spent. A cheap layer never pays the pool's
17372    /// wake-up and join (most Enron layers are tens of µs of work); an
17373    /// expensive one goes parallel after a few vertices. Splitting a layer
17374    /// into consecutive sub-layers is a valid order: its vertices are
17375    /// independent.
17376    fn evaluate_layer_parallel(
17377        &mut self,
17378        layer: &super::scheduler::Layer,
17379    ) -> Result<usize, ExcelError> {
17380        if layer.sequential {
17381            return self.evaluate_layer_sequential(layer);
17382        }
17383        self.resource_checkpoint(layer.vertices.len() as u64)?;
17384        let len = layer.vertices.len();
17385        let buffered = buffer_layer_writes(layer);
17386        let (probe, worth) = (PARALLEL_LAYER_PROBE, PARALLEL_LAYER_WORTH);
17387        let start = crate::instant::FzInstant::now();
17388        let mut pos = 0usize;
17389        let mut step = 1usize;
17390        while pos < len {
17391            if pos > 0 {
17392                let elapsed = start.elapsed();
17393                // Rate so far (ns per vertex) and the rest at that rate.
17394                let per_vertex = elapsed.as_nanos() / pos as u128 + 1;
17395                let rest_estimate = per_vertex * (len - pos) as u128;
17396                if len - pos >= 2 && (elapsed >= probe || rest_estimate >= worth.as_nanos()) {
17397                    let rest = layer.sub_layer(pos, len);
17398                    // Expensive members (a SUMIF over a table) parallelize
17399                    // one per task; cheap ones keep runs of 8 together.
17400                    let min_chunk = if per_vertex >= EXPENSIVE_VERTEX_NS {
17401                        1
17402                    } else {
17403                        8
17404                    };
17405                    return Ok(pos + self.evaluate_layer_parallel_effects(&rest, min_chunk)?);
17406                }
17407                // Next slice: double, but no more than the rest of the probe
17408                // at the rate so far (a slice must not overshoot it).
17409                let fit = (probe.saturating_sub(elapsed).as_nanos() / per_vertex) as usize + 1;
17410                step = step.saturating_mul(2).min(fit);
17411            }
17412            let end = (pos + step).min(len);
17413            let slice = layer.sub_layer(pos, end);
17414            // A slice stops at the probe's end even if its members turn out
17415            // far more expensive than the rate so far predicted.
17416            pos += self.evaluate_layer_units_until(
17417                &slice,
17418                None,
17419                None,
17420                None,
17421                buffered,
17422                Some(start + probe),
17423            )?;
17424        }
17425        Ok(len)
17426    }
17427
17428    fn evaluate_layer_parallel_with_delta(
17429        &mut self,
17430        layer: &super::scheduler::Layer,
17431        delta: &mut DeltaCollector,
17432    ) -> Result<usize, ExcelError> {
17433        if layer.sequential {
17434            return self.evaluate_layer_sequential_with_delta(layer, delta);
17435        }
17436        self.resource_checkpoint(layer.vertices.len() as u64)?;
17437        self.evaluate_layer_parallel_with_delta_effects(layer, delta)
17438    }
17439
17440    /// Evaluate a layer in parallel with cancellation support
17441    fn evaluate_layer_parallel_cancellable(
17442        &mut self,
17443        layer: &super::scheduler::Layer,
17444        cancel_flag: &AtomicBool,
17445    ) -> Result<usize, ExcelError> {
17446        if layer.sequential {
17447            return self.evaluate_layer_sequential_cancellable(layer, cancel_flag);
17448        }
17449        self.resource_checkpoint(layer.vertices.len() as u64)?;
17450        self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
17451    }
17452
17453    /// Evaluate a single vertex without mutating the graph (for parallel evaluation)
17454    fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17455        // Check if vertex exists
17456        if !self.graph.vertex_exists(vertex_id) {
17457            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17458                .with_message(format!("Vertex not found: {vertex_id:?}")));
17459        }
17460
17461        // Get vertex kind and check if it needs evaluation
17462        let kind = self.graph.get_vertex_kind(vertex_id);
17463        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17464
17465        let view = match kind {
17466            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17467                if let Some(view) = self.graph.formula_view(vertex_id) {
17468                    view
17469                } else {
17470                    return Ok(LiteralValue::Number(0.0));
17471                }
17472            }
17473            VertexKind::Empty | VertexKind::Cell => {
17474                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17475                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17476                    let row = cell_ref.coord.row() + 1;
17477                    let col = cell_ref.coord.col() + 1;
17478                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17479                        return Ok(v);
17480                    }
17481                }
17482                return Ok(LiteralValue::Number(0.0));
17483            }
17484            VertexKind::NamedScalar => {
17485                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17486                    ExcelError::new(ExcelErrorKind::Name)
17487                        .with_message("Named range metadata missing".to_string())
17488                })?;
17489
17490                return match &named_range.definition {
17491                    NamedDefinition::Cell(cell_ref) => {
17492                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17493                        Ok(self
17494                            .read_cell_value(
17495                                sheet_name,
17496                                cell_ref.coord.row() + 1,
17497                                cell_ref.coord.col() + 1,
17498                            )
17499                            .unwrap_or(LiteralValue::Empty))
17500                    }
17501                    NamedDefinition::Literal(v) => Ok(v.clone()),
17502                    NamedDefinition::Formula { ast, .. } => {
17503                        let context_sheet = match named_range.scope {
17504                            NameScope::Sheet(id) => id,
17505                            NameScope::Workbook => sheet_id,
17506                        };
17507                        let sheet_name = self.graph.sheet_name(context_sheet);
17508                        let cell_ref = self
17509                            .graph
17510                            .get_cell_ref(vertex_id)
17511                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17512                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17513                            .with_resolved_functions(&self.resolved_functions)
17514                            .with_arena_cells(self);
17515                        interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
17516                    }
17517                    NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17518                        .with_message("Range-valued name evaluated as scalar".to_string())),
17519                };
17520            }
17521            VertexKind::NamedArray => {
17522                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17523                    ExcelError::new(ExcelErrorKind::Name)
17524                        .with_message("Named range metadata missing".to_string())
17525                })?;
17526
17527                return match &named_range.definition {
17528                    NamedDefinition::Range(range_ref) => {
17529                        if range_ref.start.sheet_id != range_ref.end.sheet_id {
17530                            return Err(ExcelError::new(ExcelErrorKind::Ref)
17531                                .with_message("Named range cannot span sheets".to_string()));
17532                        }
17533                        let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17534                        let sr0 = range_ref.start.coord.row();
17535                        let sc0 = range_ref.start.coord.col();
17536                        let er0 = range_ref.end.coord.row();
17537                        let ec0 = range_ref.end.coord.col();
17538                        if sr0 > er0 || sc0 > ec0 {
17539                            return Err(ExcelError::new(ExcelErrorKind::Ref)
17540                                .with_message("Invalid named range bounds".to_string()));
17541                        }
17542
17543                        let h = (er0 - sr0 + 1) as usize;
17544                        let w = (ec0 - sc0 + 1) as usize;
17545                        let cell_count = (h as u64).saturating_mul(w as u64);
17546                        if cell_count > self.config.spill.max_spill_cells as u64 {
17547                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17548                                "Named range too large to materialize as an array".to_string(),
17549                            ));
17550                        }
17551
17552                        // Read numeric serials, resolving the sheet once.
17553                        let sheet_id = range_ref.start.sheet_id;
17554                        let asheet = self.arrow_sheets.sheet(sheet_name);
17555                        let mut rows = Vec::with_capacity(h);
17556                        for r0 in sr0..=er0 {
17557                            let mut row = Vec::with_capacity(w);
17558                            for c0 in sc0..=ec0 {
17559                                row.push(
17560                                    self.read_cell_formatted_in(sheet_id, asheet, r0 + 1, c0 + 1)
17561                                        .0,
17562                                );
17563                            }
17564                            rows.push(row);
17565                        }
17566                        Ok(LiteralValue::Array(rows))
17567                    }
17568                    NamedDefinition::Cell(cell_ref) => {
17569                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17570                        let row = cell_ref.coord.row() + 1;
17571                        let col = cell_ref.coord.col() + 1;
17572                        let v = self
17573                            .read_cell_value(sheet_name, row, col)
17574                            .unwrap_or(LiteralValue::Empty);
17575                        Ok(LiteralValue::Array(vec![vec![v]]))
17576                    }
17577                    NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
17578                    NamedDefinition::Formula { ast, .. } => {
17579                        let context_sheet = match named_range.scope {
17580                            NameScope::Sheet(id) => id,
17581                            NameScope::Workbook => sheet_id,
17582                        };
17583                        let sheet_name = self.graph.sheet_name(context_sheet);
17584                        let cell_ref = self
17585                            .graph
17586                            .get_cell_ref(vertex_id)
17587                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17588                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17589                            .with_resolved_functions(&self.resolved_functions)
17590                            .with_arena_cells(self);
17591                        match interpreter.evaluate_ast(ast) {
17592                            Ok(cv) => {
17593                                let v = cv.into_literal();
17594                                match v {
17595                                    LiteralValue::Array(_) => Ok(v),
17596                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
17597                                }
17598                            }
17599                            Err(err) => Ok(LiteralValue::Error(err)),
17600                        }
17601                    }
17602                };
17603            }
17604            VertexKind::InfiniteRange
17605            | VertexKind::Range
17606            | VertexKind::External
17607            | VertexKind::Table => {
17608                // Not directly evaluatable here.
17609                return Ok(LiteralValue::Number(0.0));
17610            }
17611        };
17612
17613        // The interpreter uses a reference to the engine as the context
17614        let sheet_name = self.graph.sheet_name(sheet_id);
17615        let cell_ref = self
17616            .graph
17617            .get_cell_ref(vertex_id)
17618            .expect("cell ref for vertex");
17619        if let Some(result) =
17620            self.freshness_evaluate_recorded(vertex_id, sheet_name, cell_ref, view)
17621        {
17622            return result;
17623        }
17624        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17625            .with_resolved_functions(&self.resolved_functions)
17626            .with_arena_cells(self);
17627
17628        interpreter
17629            .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
17630            .map(|cv| {
17631                let format = cv.format_id();
17632                self.record_derived_format(vertex_id, format);
17633                self.materialize_formula_result(vertex_id, cv)
17634            })
17635            .or_else(|error| self.fit_formula_error(vertex_id, error))
17636    }
17637
17638    /// Get access to the shared thread pool for parallel evaluation
17639    pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
17640        self.thread_pool.as_ref()
17641    }
17642}
17643
17644#[derive(Default)]
17645struct RowBoundsCache {
17646    snapshot: u64,
17647    // key: (sheet_id, col_idx)
17648    map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
17649}
17650
17651impl RowBoundsCache {
17652    fn new(snapshot: u64) -> Self {
17653        Self {
17654            snapshot,
17655            map: Default::default(),
17656        }
17657    }
17658    fn get_row_bounds(
17659        &self,
17660        sheet_id: SheetId,
17661        col_idx: usize,
17662        snapshot: u64,
17663    ) -> Option<(Option<u32>, Option<u32>)> {
17664        if self.snapshot != snapshot {
17665            return None;
17666        }
17667        self.map.get(&(sheet_id as u32, col_idx)).copied()
17668    }
17669    fn put_row_bounds(
17670        &mut self,
17671        sheet_id: SheetId,
17672        col_idx: usize,
17673        snapshot: u64,
17674        bounds: (Option<u32>, Option<u32>),
17675    ) {
17676        if self.snapshot != snapshot {
17677            self.snapshot = snapshot;
17678            self.map.clear();
17679        }
17680        self.map.insert((sheet_id as u32, col_idx), bounds);
17681    }
17682}
17683
17684struct UsedAxisBoundsCache {
17685    snapshot: u64,
17686    row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17687    col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17688    #[cfg(test)]
17689    row_hits: std::sync::atomic::AtomicUsize,
17690    #[cfg(test)]
17691    row_misses: std::sync::atomic::AtomicUsize,
17692    #[cfg(test)]
17693    col_hits: std::sync::atomic::AtomicUsize,
17694    #[cfg(test)]
17695    col_misses: std::sync::atomic::AtomicUsize,
17696}
17697
17698impl UsedAxisBoundsCache {
17699    fn new(snapshot: u64) -> Self {
17700        Self {
17701            snapshot,
17702            row_bounds_by_col_span: Default::default(),
17703            col_bounds_by_row_span: Default::default(),
17704            #[cfg(test)]
17705            row_hits: std::sync::atomic::AtomicUsize::new(0),
17706            #[cfg(test)]
17707            row_misses: std::sync::atomic::AtomicUsize::new(0),
17708            #[cfg(test)]
17709            col_hits: std::sync::atomic::AtomicUsize::new(0),
17710            #[cfg(test)]
17711            col_misses: std::sync::atomic::AtomicUsize::new(0),
17712        }
17713    }
17714
17715    fn reset_for_snapshot(&mut self, snapshot: u64) {
17716        if self.snapshot != snapshot {
17717            self.snapshot = snapshot;
17718            self.row_bounds_by_col_span.clear();
17719            self.col_bounds_by_row_span.clear();
17720        }
17721    }
17722
17723    fn get_row_bounds(
17724        &self,
17725        sheet_id: SheetId,
17726        start_col: u32,
17727        end_col: u32,
17728        snapshot: u64,
17729    ) -> Option<Option<(u32, u32)>> {
17730        if self.snapshot != snapshot {
17731            return None;
17732        }
17733        let cached = self
17734            .row_bounds_by_col_span
17735            .get(&(sheet_id, start_col, end_col))
17736            .copied();
17737        #[cfg(test)]
17738        if cached.is_some() {
17739            self.row_hits.fetch_add(1, Ordering::Relaxed);
17740        }
17741        cached
17742    }
17743
17744    fn put_row_bounds(
17745        &mut self,
17746        sheet_id: SheetId,
17747        start_col: u32,
17748        end_col: u32,
17749        snapshot: u64,
17750        bounds: Option<(u32, u32)>,
17751    ) {
17752        self.reset_for_snapshot(snapshot);
17753        self.row_bounds_by_col_span
17754            .insert((sheet_id, start_col, end_col), bounds);
17755        #[cfg(test)]
17756        self.row_misses.fetch_add(1, Ordering::Relaxed);
17757    }
17758
17759    fn get_col_bounds(
17760        &self,
17761        sheet_id: SheetId,
17762        start_row: u32,
17763        end_row: u32,
17764        snapshot: u64,
17765    ) -> Option<Option<(u32, u32)>> {
17766        if self.snapshot != snapshot {
17767            return None;
17768        }
17769        let cached = self
17770            .col_bounds_by_row_span
17771            .get(&(sheet_id, start_row, end_row))
17772            .copied();
17773        #[cfg(test)]
17774        if cached.is_some() {
17775            self.col_hits.fetch_add(1, Ordering::Relaxed);
17776        }
17777        cached
17778    }
17779
17780    fn put_col_bounds(
17781        &mut self,
17782        sheet_id: SheetId,
17783        start_row: u32,
17784        end_row: u32,
17785        snapshot: u64,
17786        bounds: Option<(u32, u32)>,
17787    ) {
17788        self.reset_for_snapshot(snapshot);
17789        self.col_bounds_by_row_span
17790            .insert((sheet_id, start_row, end_row), bounds);
17791        #[cfg(test)]
17792        self.col_misses.fetch_add(1, Ordering::Relaxed);
17793    }
17794}
17795
17796// Phase 2 shim: in-process spill manager delegating to current graph methods.
17797#[derive(Default)]
17798pub struct ShimSpillManager {
17799    region_locks: RegionLockManager,
17800    pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
17801}
17802
17803impl ShimSpillManager {
17804    pub(crate) fn reserve(
17805        &mut self,
17806        owner: VertexId,
17807        anchor_cell: CellRef,
17808        shape: SpillShape,
17809        _meta: SpillMeta,
17810    ) -> Result<(), ExcelError> {
17811        // Derive region from anchor + shape; enforce in-flight exclusivity only.
17812        let region = crate::engine::spill::Region {
17813            sheet_id: anchor_cell.sheet_id as u32,
17814            row_start: anchor_cell.coord.row(),
17815            row_end: anchor_cell
17816                .coord
17817                .row()
17818                .saturating_add(shape.rows)
17819                .saturating_sub(1),
17820            col_start: anchor_cell.coord.col(),
17821            col_end: anchor_cell
17822                .coord
17823                .col()
17824                .saturating_add(shape.cols)
17825                .saturating_sub(1),
17826        };
17827        match self.region_locks.reserve(region, owner) {
17828            Ok(id) => {
17829                if id != 0 {
17830                    self.active_locks.insert(owner, id);
17831                }
17832                Ok(())
17833            }
17834            Err(e) => Err(e),
17835        }
17836    }
17837
17838    /// Release any in-flight region reservation still held for `owner`.
17839    ///
17840    /// Reservations are normally released on commit/rollback, but if an anchor is
17841    /// abandoned without committing (e.g. cycle detection stamps it with #CIRC), a
17842    /// stale reservation could remain. This is a no-op when nothing is held.
17843    pub(crate) fn release_owner(&mut self, owner: VertexId) {
17844        if let Some(id) = self.active_locks.remove(&owner) {
17845            self.region_locks.release(id);
17846        }
17847    }
17848
17849    pub(crate) fn commit_array_with_value_probe<F>(
17850        &mut self,
17851        graph: &mut DependencyGraph,
17852        anchor_vertex: VertexId,
17853        targets: &[CellRef],
17854        rows: Vec<Vec<LiteralValue>>,
17855        mut value_probe: F,
17856    ) -> Result<(), ExcelError>
17857    where
17858        F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
17859    {
17860        use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
17861
17862        // Re-run plan on concrete targets before committing to respect blockers.
17863        // This plan checks formula/spill ownership in the graph, but when the graph value cache
17864        // is disabled (Arrow-canonical mode), it cannot see non-empty value blockers.
17865        let plan_res = graph.plan_spill_region(anchor_vertex, targets);
17866        if let Err(e) = plan_res {
17867            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17868                self.region_locks.release(id);
17869            }
17870            return Err(e);
17871        }
17872
17873        if !graph.value_cache_enabled() {
17874            // Compute expected spill shape from the target rectangle for diagnostics.
17875            let (expected_rows, expected_cols) = if targets.is_empty() {
17876                (0u32, 0u32)
17877            } else {
17878                let mut min_r = u32::MAX;
17879                let mut max_r = 0u32;
17880                let mut min_c = u32::MAX;
17881                let mut max_c = 0u32;
17882                for cell in targets {
17883                    let r = cell.coord.row();
17884                    let c = cell.coord.col();
17885                    min_r = min_r.min(r);
17886                    max_r = max_r.max(r);
17887                    min_c = min_c.min(c);
17888                    max_c = max_c.max(c);
17889                }
17890                (
17891                    max_r.saturating_sub(min_r).saturating_add(1),
17892                    max_c.saturating_sub(min_c).saturating_add(1),
17893                )
17894            };
17895
17896            let anchor_cell = graph
17897                .get_cell_ref(anchor_vertex)
17898                .expect("anchor cell ref for spill commit");
17899
17900            for cell in targets {
17901                // Never treat the anchor as a blocker.
17902                if *cell == anchor_cell {
17903                    continue;
17904                }
17905                // Skip cells already known to be owned by a spill; plan() handled spill conflicts.
17906                if graph.spill_registry_anchor_for_cell(*cell).is_some() {
17907                    continue;
17908                }
17909                // Skip formula vertices in the target region; plan() handled them (or allowed).
17910                if let Some(vid) = graph.get_vertex_id_for_address(cell)
17911                    && vid != anchor_vertex
17912                {
17913                    match graph.get_vertex_kind(vid) {
17914                        crate::engine::vertex::VertexKind::FormulaScalar
17915                        | crate::engine::vertex::VertexKind::FormulaArray => {
17916                            // plan() already approved allowed overwrites.
17917                            continue;
17918                        }
17919                        _ => {}
17920                    }
17921                }
17922
17923                if let Some(v) = value_probe(graph, cell)
17924                    && !matches!(v, LiteralValue::Empty)
17925                {
17926                    if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17927                        self.region_locks.release(id);
17928                    }
17929                    return Err(ExcelError::new(ExcelErrorKind::Spill)
17930                        .with_message("BlockedByValue")
17931                        .with_extra(ExcelErrorExtra::Spill {
17932                            expected_rows,
17933                            expected_cols,
17934                        }));
17935                }
17936            }
17937        }
17938
17939        let commit_res = graph.commit_spill_region_atomic_with_fault(
17940            anchor_vertex,
17941            targets.to_vec(),
17942            rows,
17943            None,
17944        );
17945        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17946            self.region_locks.release(id);
17947        }
17948        commit_res.map(|_| ())
17949    }
17950
17951    /// Commit a spill and mirror all written cells into Arrow overlay via the owning engine.
17952    pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
17953        &mut self,
17954        engine: &mut Engine<R>,
17955        anchor_vertex: VertexId,
17956        targets: &[CellRef],
17957        rows: Vec<Vec<LiteralValue>>,
17958    ) -> Result<(), ExcelError> {
17959        if let Err(error) = engine.guard_pending_spill_commit(anchor_vertex, targets) {
17960            self.release_owner(anchor_vertex);
17961            return Err(error);
17962        }
17963        // Re-run plan on concrete targets before committing to respect blockers.
17964        let plan_res = engine.graph.plan_spill_region(anchor_vertex, targets);
17965        if let Err(e) = plan_res {
17966            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17967                self.region_locks.release(id);
17968            }
17969            return Err(e);
17970        }
17971
17972        let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
17973            anchor_vertex,
17974            targets.to_vec(),
17975            rows.clone(),
17976            None,
17977        );
17978        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17979            self.region_locks.release(id);
17980        }
17981        commit_res.map(|_| ())?;
17982        engine
17983            .blocked_pending_spills
17984            .retain(|entry| entry.0 != anchor_vertex);
17985        engine.spill_blocker_waiters.forget(anchor_vertex);
17986
17987        // Mirror into Arrow overlay when enabled
17988        if engine.config.arrow_storage_enabled
17989            && engine.config.delta_overlay_enabled
17990            && engine.config.write_formula_overlay_enabled
17991        {
17992            // Expect targets to be a contiguous rectangle row-major starting at some anchor
17993            for (idx, cell) in targets.iter().enumerate() {
17994                let (r_off, c_off) = {
17995                    if rows.is_empty() || rows[0].is_empty() {
17996                        (0usize, 0usize)
17997                    } else {
17998                        let width = rows[0].len();
17999                        (idx / width, idx % width)
18000                    }
18001                };
18002                let v = rows
18003                    .get(r_off)
18004                    .and_then(|r| r.get(c_off))
18005                    .cloned()
18006                    .unwrap_or(LiteralValue::Empty);
18007                let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
18008                engine.mirror_value_to_computed_overlay(
18009                    &sheet_name,
18010                    cell.coord.row() + 1,
18011                    cell.coord.col() + 1,
18012                    &v,
18013                );
18014            }
18015        }
18016        Ok(())
18017    }
18018}
18019
18020impl<R> Engine<R>
18021where
18022    R: EvaluationContext,
18023{
18024    fn resolve_shared_ref(
18025        &self,
18026        reference: &ReferenceType,
18027        current_sheet: &str,
18028    ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
18029        use formualizer_common::{
18030            SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
18031            SheetRef as SharedRef,
18032        };
18033
18034        // Preserve anchor flags from the parsed reference when possible.
18035        let sr = match reference {
18036            ReferenceType::Cell {
18037                sheet,
18038                row,
18039                col,
18040                row_abs,
18041                col_abs,
18042            } => {
18043                let row0 = row
18044                    .checked_sub(1)
18045                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18046                let col0 = col
18047                    .checked_sub(1)
18048                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18049                let sheet_loc = match sheet.as_deref() {
18050                    Some(name) => SheetLocator::from_name(name),
18051                    None => SheetLocator::Current,
18052                };
18053                let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
18054                SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
18055            }
18056            ReferenceType::Range {
18057                sheet,
18058                start_row,
18059                start_col,
18060                end_row,
18061                end_col,
18062                start_row_abs,
18063                start_col_abs,
18064                end_row_abs,
18065                end_col_abs,
18066            } => {
18067                let sheet_loc = match sheet.as_deref() {
18068                    Some(name) => SheetLocator::from_name(name),
18069                    None => SheetLocator::Current,
18070                };
18071                let sr = start_row
18072                    .map(|r| {
18073                        r.checked_sub(1)
18074                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18075                    })
18076                    .transpose()?;
18077                let sc = start_col
18078                    .map(|c| {
18079                        c.checked_sub(1)
18080                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18081                    })
18082                    .transpose()?;
18083                let er = end_row
18084                    .map(|r| {
18085                        r.checked_sub(1)
18086                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18087                    })
18088                    .transpose()?;
18089                let ec = end_col
18090                    .map(|c| {
18091                        c.checked_sub(1)
18092                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18093                    })
18094                    .transpose()?;
18095                let range = SharedRangeRef::from_parts(
18096                    sheet_loc,
18097                    sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
18098                    sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
18099                    er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
18100                    ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
18101                )
18102                .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
18103                SharedRef::Range(range)
18104            }
18105            _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18106        };
18107
18108        let current_id = self
18109            .graph
18110            .sheet_id(current_sheet)
18111            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18112
18113        let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
18114            match loc {
18115                SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
18116                SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
18117                SheetLocator::Name(name) => {
18118                    let n = name.as_ref();
18119                    self.graph
18120                        .sheet_id(n)
18121                        .map(SheetLocator::Id)
18122                        .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18123                }
18124            }
18125        };
18126
18127        match sr {
18128            SharedRef::Cell(cell) => {
18129                let owned = cell.into_owned();
18130                let sheet = resolve_loc(owned.sheet)?;
18131                Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
18132            }
18133            SharedRef::Range(range) => {
18134                let owned = range.into_owned();
18135                let sheet = resolve_loc(owned.sheet)?;
18136                Ok(SharedRef::Range(SharedRangeRef {
18137                    sheet,
18138                    start_row: owned.start_row,
18139                    start_col: owned.start_col,
18140                    end_row: owned.end_row,
18141                    end_col: owned.end_col,
18142                }))
18143            }
18144        }
18145    }
18146}
18147
18148// Implement the resolver traits for the Engine.
18149// This allows the interpreter to resolve references by querying the engine's graph.
18150impl<R> crate::traits::ReferenceResolver for Engine<R>
18151where
18152    R: EvaluationContext,
18153{
18154    fn resolve_cell_reference(
18155        &self,
18156        sheet: Option<&str>,
18157        row: u32,
18158        col: u32,
18159    ) -> Result<LiteralValue, ExcelError> {
18160        // This context-free trait method has no knowledge of the formula's
18161        // current sheet, so an unqualified (`None`) reference cannot be resolved
18162        // here. Previously this fell back to `default_sheet_name()`, which leaked
18163        // the reference onto an unrelated sheet (issue #110). Interpreter paths
18164        // already qualify references with the current sheet before reaching this
18165        // method (see `Interpreter::implicit_intersection_from_reference`), and
18166        // the sheet-aware scalar path goes through `resolve_cell_reference_value`
18167        // with an explicit `current_sheet`. Returning #REF! for an unqualified
18168        // reference here surfaces the missing context instead of silently
18169        // returning data from the wrong sheet.
18170        let Some(sheet_name) = sheet else {
18171            return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18172                "Unqualified cell reference resolved without sheet context".to_string(),
18173            ));
18174        };
18175        // Prefer engine's unified accessor which consults Arrow store for base values
18176        // and falls back to graph for formulas and stored values.
18177        if let Some(v) = self.read_cell_value(sheet_name, row, col) {
18178            Ok(v)
18179        } else {
18180            // Excel semantics: empty cell coerces to 0 in numeric contexts
18181            Ok(LiteralValue::Number(0.0))
18182        }
18183    }
18184}
18185
18186impl<R> crate::traits::RangeResolver for Engine<R>
18187where
18188    R: EvaluationContext,
18189{
18190    fn resolve_range_reference(
18191        &self,
18192        sheet: Option<&str>,
18193        sr: Option<u32>,
18194        sc: Option<u32>,
18195        er: Option<u32>,
18196        ec: Option<u32>,
18197    ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
18198        // For now, delegate range resolution to the external resolver.
18199        // A future optimization could be to handle this within the graph.
18200        self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
18201    }
18202}
18203
18204impl<R> crate::traits::NamedRangeResolver for Engine<R>
18205where
18206    R: EvaluationContext,
18207{
18208    fn resolve_named_range_reference(
18209        &self,
18210        name: &str,
18211    ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
18212        self.resolver.resolve_named_range_reference(name)
18213    }
18214}
18215
18216impl<R> crate::traits::TableResolver for Engine<R>
18217where
18218    R: EvaluationContext,
18219{
18220    fn resolve_table_reference(
18221        &self,
18222        tref: &formualizer_parse::parser::TableReference,
18223    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
18224        self.resolver.resolve_table_reference(tref)
18225    }
18226}
18227
18228impl<R> crate::traits::SourceResolver for Engine<R>
18229where
18230    R: EvaluationContext,
18231{
18232    fn source_scalar_version(&self, name: &str) -> Option<u64> {
18233        self.resolver.source_scalar_version(name)
18234    }
18235
18236    fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
18237        self.resolver.resolve_source_scalar(name)
18238    }
18239
18240    fn source_table_version(&self, name: &str) -> Option<u64> {
18241        self.resolver.source_table_version(name)
18242    }
18243
18244    fn resolve_source_table(
18245        &self,
18246        name: &str,
18247    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
18248        self.resolver.resolve_source_table(name)
18249    }
18250}
18251
18252// The Engine is a Resolver because it implements the constituent traits.
18253impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
18254
18255// The Engine provides functions by delegating to its internal resolver.
18256impl<R: EvaluationContext> crate::interpreter::ArenaCellResolver for Engine<R> {
18257    fn resolve_arena_range(
18258        &self,
18259        reference: &crate::engine::arena::CompactRefType,
18260        current: SheetId,
18261        row_delta: i64,
18262        col_delta: i64,
18263    ) -> Result<RangeView<'_>, ExcelError> {
18264        use crate::engine::arena::{CompactRefType, SheetKey};
18265        let CompactRefType::Range {
18266            sheet,
18267            start_row,
18268            start_col,
18269            end_row,
18270            end_col,
18271            start_row_abs,
18272            start_col_abs,
18273            end_row_abs,
18274            end_col_abs,
18275        } = reference
18276        else {
18277            return Err(ExcelError::new(ExcelErrorKind::Ref));
18278        };
18279        let sheet_id = match sheet {
18280            None => current,
18281            Some(SheetKey::Id(id)) => *id,
18282            Some(SheetKey::Name(id)) => self
18283                .graph
18284                .sheet_id(self.graph.data_store().resolve_ast_string(*id))
18285                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?,
18286        };
18287        if self.graph.sheet_reg().name(sheet_id).is_empty() {
18288            return Err(ExcelError::new(ExcelErrorKind::Ref));
18289        }
18290        let axis = |value, sentinel, delta, absolute| {
18291            if value == sentinel {
18292                Ok(None)
18293            } else {
18294                crate::interpreter::shift_axis_for_offset(value, delta, absolute).map(Some)
18295            }
18296        };
18297        let sr = axis(*start_row, 0, row_delta, *start_row_abs)?;
18298        let sc = axis(*start_col, 0, col_delta, *start_col_abs)?;
18299        let er = axis(*end_row, u32::MAX, row_delta, *end_row_abs)?;
18300        let ec = axis(*end_col, u32::MAX, col_delta, *end_col_abs)?;
18301        if sr.zip(er).is_some_and(|(s, e)| s > e) || sc.zip(ec).is_some_and(|(s, e)| s > e) {
18302            return Err(ExcelError::new(ExcelErrorKind::Ref));
18303        }
18304        self.resolve_range_coordinates(sheet_id, sr, sc, er, ec)
18305    }
18306
18307    fn resolve_arena_cell(
18308        &self,
18309        sheet: Option<crate::engine::arena::SheetKey>,
18310        current: crate::SheetId,
18311        row: u32,
18312        col: u32,
18313    ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
18314        use crate::engine::arena::SheetKey;
18315        let (sheet_id, named_arrow) = match sheet {
18316            None => (current, None),
18317            Some(SheetKey::Id(id)) => {
18318                if self.graph.sheet_reg().name(id).is_empty() {
18319                    return Err(ExcelError::new(ExcelErrorKind::Ref));
18320                }
18321                (id, None)
18322            }
18323            Some(SheetKey::Name(id)) => {
18324                let (sheet_id, arrow) = self
18325                    .resolved_sheet_names
18326                    .get(&id)
18327                    .copied()
18328                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18329                (sheet_id, Some(arrow))
18330            }
18331        };
18332        let arrow = named_arrow.unwrap_or_else(|| {
18333            self.resolved_arrow_sheets
18334                .get(usize::from(sheet_id))
18335                .copied()
18336                .flatten()
18337        });
18338        let asheet = arrow
18339            .and_then(|index| self.arrow_sheets.sheets.get(index))
18340            .or_else(|| {
18341                // Graph-only sheets can acquire Arrow storage when an earlier layer
18342                // commits its first formula result. The pass snapshot cannot hold an
18343                // index for that storage yet; retain the original lookup in this case.
18344                let name = match sheet {
18345                    Some(SheetKey::Name(id)) => self.graph.data_store().resolve_ast_string(id),
18346                    _ => self.graph.sheet_name(sheet_id),
18347                };
18348                self.arrow_sheets.sheet(name)
18349            });
18350        Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
18351    }
18352}
18353
18354impl<R> crate::traits::FunctionProvider for Engine<R>
18355where
18356    R: EvaluationContext,
18357{
18358    fn planning_semantic_revision(&self) -> Option<u64> {
18359        self.resolver.planning_semantic_revision()
18360    }
18361
18362    fn get_function(
18363        &self,
18364        prefix: &str,
18365        name: &str,
18366    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
18367        self.resolver.get_function(prefix, name)
18368    }
18369
18370    fn get_function_for_planning(
18371        &self,
18372        prefix: &str,
18373        name: &str,
18374    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
18375        self.resolver.get_function_for_planning(prefix, name)
18376    }
18377}
18378
18379impl<R> Engine<R>
18380where
18381    R: EvaluationContext,
18382{
18383    fn resolve_range_coordinates(
18384        &self,
18385        sheet_id: SheetId,
18386        sr: Option<u32>,
18387        sc: Option<u32>,
18388        er: Option<u32>,
18389        ec: Option<u32>,
18390    ) -> Result<RangeView<'_>, ExcelError> {
18391        let sheet_name = self.graph.sheet_name(sheet_id);
18392        let extent = resolve_used_extent_with_fallback(
18393            OpenRangeBounds {
18394                start_row: sr,
18395                start_column: sc,
18396                end_row: er,
18397                end_column: ec,
18398            },
18399            ExtentPolicy::EvaluationCompat {
18400                fallback_row: None,
18401                fallback_column: None,
18402            },
18403            || {
18404                self.sheet_bounds(sheet_name)
18405                    .map(|_| self.config.max_open_ended_rows)
18406            },
18407            || {
18408                self.sheet_bounds(sheet_name)
18409                    .map(|_| self.config.max_open_ended_cols)
18410            },
18411            |first, last| self.used_rows_for_columns(sheet_name, first, last),
18412            |first, last| self.used_cols_for_rows(sheet_name, first, last),
18413        );
18414        let (sr, sc, er, ec) = extent
18415            .map(|e| (e.start_row, e.start_column, e.end_row, e.end_column))
18416            .unwrap_or((1, 1, 0, 0));
18417        if self.force_materialize_range_views {
18418            if er < sr || ec < sc {
18419                return Ok(RangeView::from_owned_rows(
18420                    Vec::new(),
18421                    self.config.date_system,
18422                ));
18423            }
18424            let h = (er - sr + 1) as u64;
18425            let w = (ec - sc + 1) as u64;
18426            if h.saturating_mul(w) <= self.config.spill.max_spill_cells as u64 {
18427                let mut rows = Vec::with_capacity(h as usize);
18428                for r in sr..=er {
18429                    let mut row = Vec::with_capacity(w as usize);
18430                    for c in sc..=ec {
18431                        row.push(
18432                            self.read_cell_value(sheet_name, r, c)
18433                                .unwrap_or(LiteralValue::Empty),
18434                        );
18435                    }
18436                    rows.push(row);
18437                }
18438                return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
18439            }
18440        }
18441        let Some(asheet) = self.arrow_sheet_by_id(sheet_id) else {
18442            return Ok(RangeView::from_owned_rows(
18443                Vec::new(),
18444                self.config.date_system,
18445            ));
18446        };
18447        Ok(if er < sr || ec < sc {
18448            asheet.range_view(1, 1, 0, 0)
18449        } else {
18450            asheet.range_view(
18451                sr.saturating_sub(1) as usize,
18452                sc.saturating_sub(1) as usize,
18453                er.saturating_sub(1) as usize,
18454                ec.saturating_sub(1) as usize,
18455            )
18456        })
18457    }
18458
18459    /// Semantic used coordinates exclude graph-only dependency placeholders.
18460    ///
18461    /// Non-empty base/overlay/computed cells come from Arrow storage, while
18462    /// scalar and array formulas come from graph formula kinds even before
18463    /// their results are materialized. The legacy graph fallback is omitted:
18464    /// `load_packed_to_vertex` entries are either represented by those sources
18465    /// or are `Empty` dependency placeholders, not a third value authority.
18466    pub(crate) fn semantic_used_rows_for_columns(
18467        &self,
18468        sheet: &str,
18469        start_col: u32,
18470        end_col: u32,
18471    ) -> Option<(u32, u32)> {
18472        let arrow_bounds = self
18473            .sheet_store()
18474            .sheet(sheet)
18475            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18476        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18477        Self::union_used_bounds(arrow_bounds, formula_bounds)
18478    }
18479
18480    pub(crate) fn semantic_used_cols_for_rows(
18481        &self,
18482        sheet: &str,
18483        start_row: u32,
18484        end_row: u32,
18485    ) -> Option<(u32, u32)> {
18486        let arrow_bounds = self
18487            .sheet_store()
18488            .sheet(sheet)
18489            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18490        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18491        Self::union_used_bounds(arrow_bounds, formula_bounds)
18492    }
18493}
18494
18495// Override EvaluationContext to provide thread pool access
18496impl<R> crate::traits::EvaluationContext for Engine<R>
18497where
18498    R: EvaluationContext,
18499{
18500    fn clock(&self) -> &dyn crate::timezone::ClockProvider {
18501        &self.clock
18502    }
18503
18504    fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
18505        self.thread_pool.as_ref()
18506    }
18507
18508    fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
18509        self.active_cancel_flag.clone()
18510    }
18511
18512    fn chunk_hint(&self) -> Option<usize> {
18513        // Use a simple heuristic from configuration (stripe width * height) as a default hint.
18514        let hint =
18515            (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
18516        Some(hint.clamp(1024, 1 << 20)) // clamp between 1K and ~1M
18517    }
18518
18519    fn materialized_array_cells(&self) -> u64 {
18520        self.config
18521            .evaluation_budgets
18522            .scratch
18523            .materialized_array_cells
18524            .unwrap_or(crate::engine::resource_ledger::DEFAULT_MATERIALIZED_ARRAY_CELLS)
18525    }
18526
18527    fn volatile_level(&self) -> crate::traits::VolatileLevel {
18528        self.config.volatile_level
18529    }
18530
18531    fn workbook_seed(&self) -> u64 {
18532        self.config.workbook_seed
18533    }
18534
18535    fn recalc_epoch(&self) -> u64 {
18536        self.recalc_epoch
18537    }
18538
18539    fn workbook_sheet_count(&self) -> Option<usize> {
18540        Some(self.graph.sheet_reg().active_len())
18541    }
18542
18543    fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
18544        self.graph.sheet_reg().active_position(sheet)
18545    }
18546
18547    fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
18548        self.sheet_index_by_name(current_sheet)
18549    }
18550
18551    fn inspect_reference(
18552        &self,
18553        reference: &ReferenceType,
18554        current_sheet: &str,
18555    ) -> Result<Option<ReferenceInfo>, ExcelError> {
18556        let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
18557            let sheet_id = self
18558                .graph
18559                .sheet_id(sheet_name)
18560                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18561            let sheet_index = self
18562                .graph
18563                .sheet_reg()
18564                .active_position_by_id(sheet_id)
18565                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18566            Ok((sheet_id, sheet_index))
18567        };
18568
18569        let cell_info =
18570            |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
18571                let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18572                let row0 = row
18573                    .checked_sub(1)
18574                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18575                let col0 = col
18576                    .checked_sub(1)
18577                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18578                Ok(ReferenceInfo {
18579                    first_sheet_index: Some(sheet_index),
18580                    sheet_count: Some(1),
18581                    first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18582                })
18583            };
18584
18585        let range_info = |sheet_name: &str,
18586                          start_row: Option<u32>,
18587                          start_col: Option<u32>|
18588         -> Result<ReferenceInfo, ExcelError> {
18589            let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18590            let row = start_row.unwrap_or(1);
18591            let col = start_col.unwrap_or(1);
18592            let row0 = row
18593                .checked_sub(1)
18594                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18595            let col0 = col
18596                .checked_sub(1)
18597                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18598            Ok(ReferenceInfo {
18599                first_sheet_index: Some(sheet_index),
18600                sheet_count: Some(1),
18601                first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18602            })
18603        };
18604
18605        let info = match reference {
18606            ReferenceType::Cell {
18607                sheet, row, col, ..
18608            } => {
18609                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18610                cell_info(sheet_name, *row, *col)?
18611            }
18612            ReferenceType::Range {
18613                sheet,
18614                start_row,
18615                start_col,
18616                ..
18617            } => {
18618                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18619                range_info(sheet_name, *start_row, *start_col)?
18620            }
18621            ReferenceType::Cell3D {
18622                sheet_first,
18623                sheet_last,
18624                row,
18625                col,
18626                ..
18627            } => {
18628                let first = cell_info(sheet_first, *row, *col)?;
18629                ReferenceInfo {
18630                    first_sheet_index: first.first_sheet_index,
18631                    sheet_count: self
18632                        .graph
18633                        .sheet_reg()
18634                        .active_span_len(sheet_first, sheet_last),
18635                    first_cell: first.first_cell,
18636                }
18637            }
18638            ReferenceType::Range3D {
18639                sheet_first,
18640                sheet_last,
18641                start_row,
18642                start_col,
18643                ..
18644            } => {
18645                let first = range_info(sheet_first, *start_row, *start_col)?;
18646                ReferenceInfo {
18647                    first_sheet_index: first.first_sheet_index,
18648                    sheet_count: self
18649                        .graph
18650                        .sheet_reg()
18651                        .active_span_len(sheet_first, sheet_last),
18652                    first_cell: first.first_cell,
18653                }
18654            }
18655            ReferenceType::NamedRange(name) => {
18656                let current_id = self
18657                    .graph
18658                    .sheet_id(current_sheet)
18659                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18660                let named = self
18661                    .graph
18662                    .resolve_name_entry(name, current_id)
18663                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18664                match &named.definition {
18665                    NamedDefinition::Cell(cell) => ReferenceInfo {
18666                        first_sheet_index: self
18667                            .graph
18668                            .sheet_reg()
18669                            .active_position_by_id(cell.sheet_id),
18670                        sheet_count: Some(1),
18671                        first_cell: Some(*cell),
18672                    },
18673                    NamedDefinition::Range(range) => ReferenceInfo {
18674                        first_sheet_index: self
18675                            .graph
18676                            .sheet_reg()
18677                            .active_position_by_id(range.start.sheet_id),
18678                        sheet_count: Some(1),
18679                        first_cell: Some(range.start),
18680                    },
18681                    NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
18682                        ReferenceInfo {
18683                            first_sheet_index: None,
18684                            sheet_count: None,
18685                            first_cell: None,
18686                        }
18687                    }
18688                }
18689            }
18690            ReferenceType::Table(tref) => {
18691                let table = self
18692                    .graph
18693                    .resolve_table_entry(&tref.name)
18694                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18695                ReferenceInfo {
18696                    first_sheet_index: self
18697                        .graph
18698                        .sheet_reg()
18699                        .active_position_by_id(table.range.start.sheet_id),
18700                    sheet_count: Some(1),
18701                    first_cell: Some(table.range.start),
18702                }
18703            }
18704            ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18705        };
18706
18707        Ok(Some(info))
18708    }
18709
18710    fn resolve_spill_reference(
18711        &self,
18712        anchor: &ReferenceType,
18713        current_sheet: &str,
18714    ) -> Result<ReferenceType, ExcelError> {
18715        let no_spill = || {
18716            ExcelError::new(ExcelErrorKind::Ref)
18717                .with_message("Spill reference anchor has no current spill")
18718        };
18719        // The rectangle keeps the operand's sheet qualification: an
18720        // unqualified anchor on the current sheet yields an unqualified range.
18721        let mut qualify = true;
18722        let anchor_cell = match anchor {
18723            ReferenceType::Cell {
18724                sheet, row, col, ..
18725            } => {
18726                qualify = sheet.is_some();
18727                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18728                let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(no_spill)?;
18729                if *row == 0 || *col == 0 {
18730                    return Err(no_spill());
18731                }
18732                CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true))
18733            }
18734            ReferenceType::NamedRange(name) => {
18735                let current_id = self.graph.sheet_id(current_sheet).ok_or_else(no_spill)?;
18736                let named = self
18737                    .graph
18738                    .resolve_name_entry(name, current_id)
18739                    .ok_or_else(no_spill)?;
18740                match &named.definition {
18741                    NamedDefinition::Cell(cell) => *cell,
18742                    NamedDefinition::Range(range)
18743                        if range.start.sheet_id == range.end.sheet_id
18744                            && range.start.coord.row() == range.end.coord.row()
18745                            && range.start.coord.col() == range.end.coord.col() =>
18746                    {
18747                        range.start
18748                    }
18749                    _ => {
18750                        return Err(ExcelError::new(ExcelErrorKind::Ref)
18751                            .with_message("Spill reference name must refer to a single cell"));
18752                    }
18753                }
18754            }
18755            _ => {
18756                return Err(ExcelError::new(ExcelErrorKind::Ref)
18757                    .with_message("Spill reference operand must be a single cell"));
18758            }
18759        };
18760        let vertex = self
18761            .graph
18762            .get_vertex_id_for_address(&anchor_cell)
18763            .ok_or_else(no_spill)?;
18764        // Policy: the spill operator applies only to dynamic arrays, never
18765        // to a legacy fixed-extent formula (including a single-cell one).
18766        if self.graph.fixed_single_arrays.contains(&vertex)
18767            || self.graph.fixed_array_shapes.contains_key(&vertex)
18768        {
18769            return Err(ExcelError::new(ExcelErrorKind::Ref)
18770                .with_message("Spill reference to a fixed array formula"));
18771        }
18772        let (first, last) = match self.graph.spill_extent_for_anchor(vertex) {
18773            Some(extent) => extent,
18774            // FORM211: a declared anchor (source spill identity) holding a
18775            // current scalar result is a 1x1 spill. A malformed registry
18776            // entry stays `#REF!`, as do blocked/error/empty results.
18777            None if !self.graph.spill_registry_has_anchor(vertex)
18778                && self.graph.is_current_declared_dynamic_anchor(vertex)
18779                && self.declared_anchor_holds_scalar(anchor_cell) =>
18780            {
18781                (anchor_cell, anchor_cell)
18782            }
18783            None => return Err(no_spill()),
18784        };
18785        let sheet_name = qualify.then(|| self.graph.sheet_name(first.sheet_id).to_string());
18786        let (sr, sc) = (first.coord.row() + 1, first.coord.col() + 1);
18787        let (er, ec) = (last.coord.row() + 1, last.coord.col() + 1);
18788        // The rectangle is concrete, so its bounds are absolute.
18789        Ok(if sr == er && sc == ec {
18790            ReferenceType::cell_with_abs(sheet_name, sr, sc, true, true)
18791        } else {
18792            ReferenceType::Range {
18793                sheet: sheet_name,
18794                start_row: Some(sr),
18795                start_col: Some(sc),
18796                end_row: Some(er),
18797                end_col: Some(ec),
18798                start_row_abs: true,
18799                start_col_abs: true,
18800                end_row_abs: true,
18801                end_col_abs: true,
18802            }
18803        })
18804    }
18805
18806    fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
18807        let sheet_name = self.graph.sheet_name(cell.sheet_id);
18808        if sheet_name.is_empty() {
18809            return Err(ExcelError::new(ExcelErrorKind::Ref));
18810        }
18811        let row = cell.coord.row() + 1;
18812        let col = cell.coord.col() + 1;
18813
18814        if let Some(entries) = self.staged_formulas.get(sheet_name)
18815            && let Some(text) = entries.get(row, col)
18816        {
18817            return Ok(Some(if text.starts_with('=') {
18818                text.to_owned()
18819            } else {
18820                format!("={text}")
18821            }));
18822        }
18823
18824        let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
18825            return Ok(None);
18826        };
18827        Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
18828    }
18829
18830    fn used_rows_for_columns(
18831        &self,
18832        sheet: &str,
18833        start_col: u32,
18834        end_col: u32,
18835    ) -> Option<(u32, u32)> {
18836        // Union Arrow-backed used-region with formula rows that have not been materialized yet.
18837        let sheet_id = self.graph.sheet_id(sheet)?;
18838        let snap = self.data_snapshot_id();
18839        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18840            guard
18841                .as_ref()
18842                .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
18843        }) {
18844            return cached;
18845        }
18846
18847        let arrow_bounds = self
18848            .sheet_store()
18849            .sheet(sheet)
18850            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18851        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18852        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18853            Some(bounds)
18854        } else {
18855            let sc0 = start_col.saturating_sub(1);
18856            let ec0 = end_col.saturating_sub(1);
18857            self.graph
18858                .used_row_bounds_for_columns(sheet_id, sc0, ec0)
18859                .map(|(a0, b0)| (a0 + 1, b0 + 1))
18860        };
18861
18862        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18863            guard
18864                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18865                .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
18866        }
18867
18868        computed
18869    }
18870
18871    fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
18872        // Union Arrow-backed used-region with formula columns that have not been materialized yet.
18873        let sheet_id = self.graph.sheet_id(sheet)?;
18874        let snap = self.data_snapshot_id();
18875        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18876            guard
18877                .as_ref()
18878                .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
18879        }) {
18880            return cached;
18881        }
18882
18883        let arrow_bounds = self
18884            .sheet_store()
18885            .sheet(sheet)
18886            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18887        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18888        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18889            Some(bounds)
18890        } else {
18891            let sr0 = start_row.saturating_sub(1);
18892            let er0 = end_row.saturating_sub(1);
18893            self.graph
18894                .used_col_bounds_for_rows(sheet_id, sr0, er0)
18895                .map(|(a0, b0)| (a0 + 1, b0 + 1))
18896        };
18897
18898        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18899            guard
18900                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18901                .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
18902        }
18903
18904        computed
18905    }
18906
18907    fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
18908        let _ = self.graph.sheet_id(sheet)?;
18909        // Excel-like upper bounds; we expose something finite but large.
18910        // Backends may override with real bounds.
18911        Some((1_048_576, 16_384)) // 1048576 rows, 16384 cols (XFD)
18912    }
18913
18914    fn data_snapshot_id(&self) -> u64 {
18915        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
18916    }
18917
18918    fn backend_caps(&self) -> crate::traits::BackendCaps {
18919        crate::traits::BackendCaps {
18920            streaming: true,
18921            used_region: true,
18922            write: false,
18923            tables: false,
18924            async_stream: false,
18925        }
18926    }
18927
18928    fn build_lookup_index(
18929        &self,
18930        view: &RangeView<'_>,
18931        axis: LookupAxis,
18932    ) -> Option<Arc<LookupIndex>> {
18933        self.build_lookup_index_impl(view, axis)
18934    }
18935
18936    // Flats removed
18937
18938    fn date_system(&self) -> crate::engine::DateSystem {
18939        self.config.date_system
18940    }
18941    /// New: resolve a reference into a RangeView (Phase 2 API)
18942    fn resolve_range_view<'c>(
18943        &'c self,
18944        reference: &ReferenceType,
18945        current_sheet: &str,
18946    ) -> Result<RangeView<'c>, ExcelError> {
18947        match reference {
18948            ReferenceType::External(ext) => {
18949                let name = ext.raw.as_str();
18950                match ext.kind {
18951                    formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
18952                        let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
18953                            return Err(ExcelError::new(ExcelErrorKind::Name)
18954                                .with_message(format!("Undefined name: {name}")));
18955                        };
18956                        let version = source
18957                            .version
18958                            .or_else(|| self.resolver.source_scalar_version(name));
18959                        let v = self.resolve_source_scalar_cached(name, version)?;
18960                        Ok(RangeView::from_owned_rows(
18961                            vec![vec![v]],
18962                            self.config.date_system,
18963                        ))
18964                    }
18965                    formualizer_parse::parser::ExternalRefKind::Range { .. } => {
18966                        let Some(source) = self.graph.resolve_source_table_entry(name) else {
18967                            // A deferred (whole-row/column) external range
18968                            // evaluates to #REF!, as it did before the parser
18969                            // kept it whole (see `unbound_external_range_defers`).
18970                            let kind =
18971                                if crate::engine::refs::unbound_external_range_defers(&ext.kind) {
18972                                    ExcelErrorKind::Ref
18973                                } else {
18974                                    ExcelErrorKind::Name
18975                                };
18976                            return Err(ExcelError::new(kind)
18977                                .with_message(format!("Undefined table: {name}")));
18978                        };
18979                        let version = source
18980                            .version
18981                            .or_else(|| self.resolver.source_table_version(name));
18982                        let key = (name.to_string(), version);
18983                        if let Ok(g) = self.source_cache.read()
18984                            && let Some(view) = g.external_views.get(&key)
18985                        {
18986                            return Ok(view.clone());
18987                        }
18988                        let table = self.resolve_source_table_cached(name, version)?;
18989                        let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
18990                        // Build under the write lock: concurrent readers of one
18991                        // source wait for a single materialization.
18992                        let Ok(mut g) = self.source_cache.write() else {
18993                            return self.source_table_to_range_view(table.as_ref(), &spec);
18994                        };
18995                        if let Some(view) = g.external_views.get(&key) {
18996                            return Ok(view.clone());
18997                        }
18998                        let view = self.source_table_to_range_view(table.as_ref(), &spec)?;
18999                        g.external_views.insert(key, view.clone());
19000                        Ok(view)
19001                    }
19002                }
19003            }
19004            ReferenceType::Range { .. } => {
19005                let shared = self.resolve_shared_ref(reference, current_sheet)?;
19006                let formualizer_common::SheetRef::Range(range) = shared else {
19007                    return Err(ExcelError::new(ExcelErrorKind::Ref));
19008                };
19009                // No context sheet is available here, so an unresolved locator
19010                // is #REF! rather than a guess (issue #110).
19011                let sheet_id = match range.sheet {
19012                    formualizer_common::SheetLocator::Id(id) => id,
19013                    formualizer_common::SheetLocator::Current
19014                    | formualizer_common::SheetLocator::Name(_) => {
19015                        return Err(ExcelError::new(ExcelErrorKind::Ref));
19016                    }
19017                };
19018                let bounded_range = if range.start_row.is_some()
19019                    && range.start_col.is_some()
19020                    && range.end_row.is_some()
19021                    && range.end_col.is_some()
19022                {
19023                    Some(RangeRef::try_from_shared(range.as_ref())?)
19024                } else {
19025                    None
19026                };
19027
19028                let sr = bounded_range
19029                    .as_ref()
19030                    .map(|r| r.start.coord.row() + 1)
19031                    .or_else(|| range.start_row.map(|b| b.index + 1));
19032                let sc = bounded_range
19033                    .as_ref()
19034                    .map(|r| r.start.coord.col() + 1)
19035                    .or_else(|| range.start_col.map(|b| b.index + 1));
19036                let er = bounded_range
19037                    .as_ref()
19038                    .map(|r| r.end.coord.row() + 1)
19039                    .or_else(|| range.end_row.map(|b| b.index + 1));
19040                let ec = bounded_range
19041                    .as_ref()
19042                    .map(|r| r.end.coord.col() + 1)
19043                    .or_else(|| range.end_col.map(|b| b.index + 1));
19044
19045                self.resolve_range_coordinates(sheet_id, sr, sc, er, ec)
19046            }
19047            ReferenceType::Cell { .. } => {
19048                let shared = self.resolve_shared_ref(reference, current_sheet)?;
19049                let formualizer_common::SheetRef::Cell(cell) = shared else {
19050                    return Err(ExcelError::new(ExcelErrorKind::Ref));
19051                };
19052                let addr = CellRef::try_from_shared(cell)?;
19053                let sheet_id = addr.sheet_id;
19054                let sheet_name = self.graph.sheet_name(sheet_id);
19055                let row = addr.coord.row() + 1;
19056                let col = addr.coord.col() + 1;
19057
19058                if self.force_materialize_range_views {
19059                    let v = self
19060                        .read_cell_value(sheet_name, row, col)
19061                        .unwrap_or(LiteralValue::Empty);
19062                    return Ok(RangeView::from_owned_rows(
19063                        vec![vec![v]],
19064                        self.config.date_system,
19065                    ));
19066                }
19067
19068                if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
19069                    let r0 = row.saturating_sub(1) as usize;
19070                    let c0 = col.saturating_sub(1) as usize;
19071                    let rv = asheet.range_view(r0, c0, r0, c0);
19072                    Ok(rv)
19073                } else {
19074                    let v = self
19075                        .read_cell_value(sheet_name, row, col)
19076                        .unwrap_or(LiteralValue::Empty);
19077                    Ok(RangeView::from_owned_rows(
19078                        vec![vec![v]],
19079                        self.config.date_system,
19080                    ))
19081                }
19082            }
19083            ReferenceType::NamedRange(name) => {
19084                if let Some(current_id) = self.graph.sheet_id(current_sheet)
19085                    && let Some(named) = self.graph.resolve_name_entry(name, current_id)
19086                {
19087                    match &named.definition {
19088                        NamedDefinition::Cell(cell_ref) => {
19089                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
19090                            if self.force_materialize_range_views {
19091                                let v = self
19092                                    .read_cell_value(
19093                                        sheet_name,
19094                                        cell_ref.coord.row() + 1,
19095                                        cell_ref.coord.col() + 1,
19096                                    )
19097                                    .unwrap_or(LiteralValue::Empty);
19098                                return Ok(RangeView::from_owned_rows(
19099                                    vec![vec![v]],
19100                                    self.config.date_system,
19101                                ));
19102                            } else {
19103                                let asheet = self
19104                                    .sheet_store()
19105                                    .sheet(sheet_name)
19106                                    .expect("Arrow sheet missing for named cell");
19107                                let r0 = cell_ref.coord.row() as usize;
19108                                let c0 = cell_ref.coord.col() as usize;
19109                                let rv = asheet.range_view(r0, c0, r0, c0);
19110                                return Ok(rv);
19111                            }
19112                        }
19113                        NamedDefinition::Range(range_ref) => {
19114                            let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
19115                            let sr = range_ref.start.coord.row() + 1;
19116                            let sc = range_ref.start.coord.col() + 1;
19117                            let er = range_ref.end.coord.row() + 1;
19118                            let ec = range_ref.end.coord.col() + 1;
19119                            if self.force_materialize_range_views {
19120                                let h = (er.saturating_sub(sr) + 1) as u64;
19121                                let w = (ec.saturating_sub(sc) + 1) as u64;
19122                                let cell_count = h.saturating_mul(w);
19123                                if cell_count <= self.config.spill.max_spill_cells as u64 {
19124                                    let mut rows: Vec<Vec<LiteralValue>> =
19125                                        Vec::with_capacity(h as usize);
19126                                    for r in sr..=er {
19127                                        let mut rowv: Vec<LiteralValue> =
19128                                            Vec::with_capacity(w as usize);
19129                                        for c in sc..=ec {
19130                                            rowv.push(
19131                                                self.read_cell_value(sheet_name, r, c)
19132                                                    .unwrap_or(LiteralValue::Empty),
19133                                            );
19134                                        }
19135                                        rows.push(rowv);
19136                                    }
19137                                    return Ok(RangeView::from_owned_rows(
19138                                        rows,
19139                                        self.config.date_system,
19140                                    ));
19141                                }
19142                            }
19143                            let asheet = self
19144                                .sheet_store()
19145                                .sheet(sheet_name)
19146                                .expect("Arrow sheet missing for named range");
19147                            let sr0 = range_ref.start.coord.row() as usize;
19148                            let sc0 = range_ref.start.coord.col() as usize;
19149                            let er0 = range_ref.end.coord.row() as usize;
19150                            let ec0 = range_ref.end.coord.col() as usize;
19151                            let rv = asheet.range_view(sr0, sc0, er0, ec0);
19152                            return Ok(rv);
19153                        }
19154                        NamedDefinition::Literal(v) => {
19155                            return Ok(RangeView::from_owned_rows(
19156                                vec![vec![v.clone()]],
19157                                self.config.date_system,
19158                            ));
19159                        }
19160                        NamedDefinition::Formula { ast, .. } => {
19161                            // A name that is exactly one external reference
19162                            // reads its source like the reference itself.
19163                            if let ASTNodeType::Reference {
19164                                reference: external @ ReferenceType::External(_),
19165                                ..
19166                            } = &ast.node_type
19167                            {
19168                                return self.resolve_range_view(external, current_sheet);
19169                            }
19170                            if let Some(value) = self.graph.get_value(named.vertex) {
19171                                return Ok(RangeView::from_owned_rows(
19172                                    vec![vec![value]],
19173                                    self.config.date_system,
19174                                ));
19175                            }
19176                        }
19177                    }
19178                }
19179
19180                if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
19181                    let version = source
19182                        .version
19183                        .or_else(|| self.resolver.source_scalar_version(name));
19184                    let v = self.resolve_source_scalar_cached(name, version)?;
19185                    return Ok(RangeView::from_owned_rows(
19186                        vec![vec![v]],
19187                        self.config.date_system,
19188                    ));
19189                }
19190
19191                let data = self.resolver.resolve_named_range_reference(name)?;
19192                Ok(RangeView::from_owned_rows(data, self.config.date_system))
19193            }
19194            ReferenceType::Table(tref) => {
19195                if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
19196                    let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
19197                    let asheet = self
19198                        .sheet_store()
19199                        .sheet(sheet_name)
19200                        .expect("Arrow sheet missing for table reference");
19201
19202                    let sr0 = table.range.start.coord.row() as usize;
19203                    let sc0 = table.range.start.coord.col() as usize;
19204                    let er0 = table.range.end.coord.row() as usize;
19205                    let ec0 = table.range.end.coord.col() as usize;
19206
19207                    let has_totals = table.totals_row;
19208                    let has_headers = table.header_row;
19209                    let data_sr = if has_headers {
19210                        sr0.saturating_add(1)
19211                    } else {
19212                        sr0
19213                    };
19214                    let data_er = if has_totals {
19215                        er0.saturating_sub(1)
19216                    } else {
19217                        er0
19218                    };
19219
19220                    let select = |sr: usize, sc: usize, er: usize, ec: usize| {
19221                        if sr > er || sc > ec {
19222                            asheet.range_view(1, 1, 0, 0)
19223                        } else {
19224                            asheet.range_view(sr, sc, er, ec)
19225                        }
19226                    };
19227
19228                    let av = match &tref.specifier {
19229                        None => {
19230                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19231                                "Table reference without specifier is unsupported".to_string(),
19232                            ));
19233                        }
19234                        Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
19235                            let Some(idx) = table.col_index(col) else {
19236                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19237                                    "Column refers to unknown table column".to_string(),
19238                                ));
19239                            };
19240                            let c0 = sc0 + idx;
19241                            select(data_sr, c0, data_er, c0)
19242                        }
19243                        Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
19244                            start,
19245                            end,
19246                        )) => {
19247                            let Some(si) = table.col_index(start) else {
19248                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19249                                    "Column range refers to unknown column(s)".to_string(),
19250                                ));
19251                            };
19252                            let Some(ei) = table.col_index(end) else {
19253                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19254                                    "Column range refers to unknown column(s)".to_string(),
19255                                ));
19256                            };
19257                            let (mut a, mut b) = (si, ei);
19258                            if a > b {
19259                                std::mem::swap(&mut a, &mut b);
19260                            }
19261                            let c_start = sc0 + a;
19262                            let c_end = sc0 + b;
19263                            select(data_sr, c_start, data_er, c_end)
19264                        }
19265                        Some(formualizer_parse::parser::TableSpecifier::All)
19266                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19267                            formualizer_parse::parser::SpecialItem::All,
19268                        )) => select(sr0, sc0, er0, ec0),
19269                        Some(formualizer_parse::parser::TableSpecifier::Data)
19270                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19271                            formualizer_parse::parser::SpecialItem::Data,
19272                        )) => select(data_sr, sc0, data_er, ec0),
19273                        Some(formualizer_parse::parser::TableSpecifier::Headers)
19274                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19275                            formualizer_parse::parser::SpecialItem::Headers,
19276                        )) => {
19277                            if !has_headers {
19278                                asheet.range_view(1, 1, 0, 0)
19279                            } else {
19280                                select(sr0, sc0, sr0, ec0)
19281                            }
19282                        }
19283                        Some(formualizer_parse::parser::TableSpecifier::Totals)
19284                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19285                            formualizer_parse::parser::SpecialItem::Totals,
19286                        )) => {
19287                            if !has_totals {
19288                                asheet.range_view(1, 1, 0, 0)
19289                            } else {
19290                                select(er0, sc0, er0, ec0)
19291                            }
19292                        }
19293                        Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19294                            formualizer_parse::parser::SpecialItem::ThisRow,
19295                        )) => {
19296                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19297                                "@ (This Row) requires table-aware context; not yet supported"
19298                                    .to_string(),
19299                            ));
19300                        }
19301                        Some(formualizer_parse::parser::TableSpecifier::Row(_))
19302                        | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
19303                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19304                                "Complex structured references not yet supported".to_string(),
19305                            ));
19306                        }
19307                    };
19308
19309                    return Ok(av);
19310                }
19311
19312                if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
19313                    let version = source
19314                        .version
19315                        .or_else(|| self.resolver.source_table_version(&tref.name));
19316                    let table = self.resolve_source_table_cached(&tref.name, version)?;
19317                    return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
19318                }
19319
19320                // Fallback: materialize via Resolver::resolve_range_like tranche 1.
19321                // A table nobody defines (an unbound reference kept by the
19322                // BestEffort preparation policy) is `#NAME?`, the kind Strict
19323                // reports at preparation, not the resolver's "not implemented".
19324                let boxed = self
19325                    .resolve_range_like(&ReferenceType::Table(tref.clone()))
19326                    .map_err(|e| {
19327                        if e.kind == ExcelErrorKind::NImpl {
19328                            ExcelError::new(ExcelErrorKind::Name)
19329                                .with_message(format!("Unknown table: {}", tref.name))
19330                        } else {
19331                            e
19332                        }
19333                    })?;
19334                let owned = boxed.materialise().into_owned();
19335                Ok(RangeView::from_owned_rows(owned, self.config.date_system))
19336            }
19337            ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
19338                Err(ExcelError::new(ExcelErrorKind::NImpl)
19339                    .with_message("3D references are not yet supported".to_string()))
19340            }
19341        }
19342    }
19343
19344    fn resolve_cell_format(
19345        &self,
19346        sheet: Option<&str>,
19347        row: u32,
19348        col: u32,
19349        current_sheet: &str,
19350    ) -> Option<crate::format::FormatId> {
19351        self.effective_format_id(sheet.unwrap_or(current_sheet), row, col)
19352    }
19353
19354    fn format_class(
19355        &self,
19356        format: crate::format::FormatId,
19357    ) -> Option<formualizer_common::numfmt::FormatClass> {
19358        self.format_registry.class(format).cloned()
19359    }
19360
19361    fn record_cell_derived_format(
19362        &self,
19363        sheet: &str,
19364        row: u32,
19365        col: u32,
19366        format: Option<crate::format::FormatId>,
19367    ) {
19368        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
19369            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
19370            self.record_derived_format_at(cell, format);
19371        }
19372    }
19373
19374    fn resolve_cell_reference_value(
19375        &self,
19376        sheet: Option<&str>,
19377        row: u32,
19378        col: u32,
19379        current_sheet: &str,
19380    ) -> Result<LiteralValue, ExcelError> {
19381        let sheet_name = sheet.unwrap_or(current_sheet);
19382        if self.graph.sheet_id(sheet_name).is_none() {
19383            return Err(ExcelError::new(ExcelErrorKind::Ref));
19384        }
19385        Ok(self
19386            .read_cell_value(sheet_name, row, col)
19387            .unwrap_or(LiteralValue::Empty))
19388    }
19389
19390    fn resolve_cell_reference_value_formatted(
19391        &self,
19392        sheet: Option<&str>,
19393        row: u32,
19394        col: u32,
19395        current_sheet: &str,
19396    ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
19397        // `resolve_cell_reference_value` + `resolve_cell_format` with one
19398        // sheet lookup of each kind.
19399        let sheet_name = sheet.unwrap_or(current_sheet);
19400        let Some(sheet_id) = self.graph.sheet_id(sheet_name) else {
19401            return Err(ExcelError::new(ExcelErrorKind::Ref));
19402        };
19403        let asheet = self.arrow_sheets.sheet(sheet_name);
19404        Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
19405    }
19406
19407    fn build_criteria_mask(
19408        &self,
19409        view: &RangeView<'_>,
19410        col_in_view: usize,
19411        pred: &crate::args::CriteriaPredicate,
19412    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19413        #[cfg(any(test, feature = "test-support"))]
19414        criteria_mask_test_hooks::note_mask(view.dims().0);
19415        if view.dims().1 == 0 {
19416            return None;
19417        }
19418        // If the view is logically open-ended but the backing sheet has no physical rows,
19419        // treat the mask as empty (0-len) rather than attempting to build a huge mask.
19420        let sheet_rows = view.sheet().nrows as usize;
19421        if sheet_rows == 0 || view.start_row() >= sheet_rows {
19422            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
19423        }
19424        self.criteria_mask_cache.get_or_build(
19425            view,
19426            col_in_view,
19427            pred,
19428            (
19429                self.recalc_epoch,
19430                self.topology_epoch,
19431                self.graph.topology_revision(),
19432            ),
19433            || compute_criteria_mask(view, col_in_view, pred),
19434        )
19435    }
19436
19437    fn build_row_visibility_mask(
19438        &self,
19439        view: &RangeView<'_>,
19440        mode: VisibilityMaskMode,
19441    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19442        self.build_row_visibility_mask_for_view(view, mode)
19443    }
19444
19445    fn nested_subtotal_cells(
19446        &self,
19447        view: &RangeView<'_>,
19448        include_aggregate: bool,
19449    ) -> Option<Vec<(usize, usize, usize)>> {
19450        self.nested_subtotal_cells_for_view(view, include_aggregate)
19451    }
19452}
19453
19454impl<R> Engine<R>
19455where
19456    R: EvaluationContext,
19457{
19458    fn clear_spill_projection_and_mirror(
19459        &mut self,
19460        anchor_vertex: VertexId,
19461        delta: Option<&mut DeltaCollector>,
19462    ) {
19463        let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
19464        if spill_cells.is_empty() {
19465            return;
19466        }
19467
19468        if let Some(delta) = delta
19469            && delta.mode != DeltaMode::Off
19470        {
19471            let empty = LiteralValue::Empty;
19472            for cell in spill_cells.iter() {
19473                let sheet_name = self.graph.sheet_name(cell.sheet_id);
19474                let old = self
19475                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19476                    .unwrap_or(LiteralValue::Empty);
19477                if old != empty {
19478                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19479                }
19480            }
19481        }
19482
19483        self.graph.clear_spill_region(anchor_vertex);
19484        if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
19485            self.record_structural_change(scope);
19486        }
19487
19488        if self.config.arrow_storage_enabled
19489            && self.config.delta_overlay_enabled
19490            && self.config.write_formula_overlay_enabled
19491        {
19492            let empty = LiteralValue::Empty;
19493            for cell in spill_cells.iter() {
19494                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19495                self.mirror_value_to_computed_overlay(
19496                    &sheet_name,
19497                    cell.coord.row() + 1,
19498                    cell.coord.col() + 1,
19499                    &empty,
19500                );
19501            }
19502        }
19503    }
19504
19505    /// Apply the evaluation outcome for one cyclic SCC: stamp `#CIRC!` on its
19506    /// (optionally filtered) members via `stamp_cycle_error`.
19507    ///
19508    /// This is the single per-SCC application point used by every schedule
19509    /// consumer walking `Schedule::units` (pre-work for #112, where cyclic
19510    /// SCCs will gain runtime verdicts instead of an unconditional stamp).
19511    ///
19512    /// `dirty_filter` preserves the recalc-plan quirk: when `Some(dirty)`,
19513    /// only members present in the set are stamped.
19514    ///
19515    /// Returns the number of vertices stamped (0 when a filter excludes every
19516    /// member), so callers can keep their site-specific `cycle_errors`
19517    /// accounting.
19518    fn apply_cycle_outcome(
19519        &mut self,
19520        cycle: &[VertexId],
19521        mut delta: Option<&mut DeltaCollector>,
19522        dirty_filter: Option<&FxHashSet<VertexId>>,
19523    ) -> usize {
19524        let circ_error = LiteralValue::Error(
19525            ExcelError::new(ExcelErrorKind::Circ)
19526                .with_message("Circular dependency detected".to_string()),
19527        );
19528        let mut stamped = 0usize;
19529        for &vertex_id in cycle {
19530            if let Some(filter) = dirty_filter
19531                && !filter.contains(&vertex_id)
19532            {
19533                continue;
19534            }
19535            self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
19536            stamped += 1;
19537        }
19538        stamped
19539    }
19540
19541    /// Stamp a vertex with `#CIRC!` as part of cycle handling.
19542    ///
19543    /// Unlike a bare `update_vertex_value`, this first tears down any spill the
19544    /// vertex previously anchored: it clears the spilled cells, releases the graph
19545    /// spill registry, drops any lingering region reservation, and mirrors the
19546    /// cleared cells into the computed overlay — the same teardown a normal scalar/
19547    /// error result performs (see `apply_non_array_result_from_parallel` /
19548    /// `clear_spill_projection_and_mirror`). Without this, a #CIRC stamp on a former
19549    /// spill anchor would leave stale spilled values and a reserved region behind
19550    /// (issue #111).
19551    ///
19552    /// When `delta` is provided, the cleared spill cells are recorded (by
19553    /// `clear_spill_projection_and_mirror`) and the anchor's own #CIRC change is
19554    /// recorded here, matching how other result paths emit deltas.
19555    fn stamp_cycle_error(
19556        &mut self,
19557        vertex_id: VertexId,
19558        circ_error: &LiteralValue,
19559        mut delta: Option<&mut DeltaCollector>,
19560    ) {
19561        // Tear down any previous spill projection/region before overwriting the anchor.
19562        if self.graph.spill_registry_has_anchor(vertex_id) {
19563            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
19564        }
19565        // Drop any reservation that was never committed (defensive; normally released
19566        // on the prior successful commit).
19567        self.spill_mgr.release_owner(vertex_id);
19568
19569        // Record the anchor's own #CIRC delta, like other result paths.
19570        if let Some(d) = delta
19571            && d.mode != DeltaMode::Off
19572            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
19573        {
19574            let sheet_name = self.graph.sheet_name(cell.sheet_id);
19575            let old = self
19576                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19577                .unwrap_or(LiteralValue::Empty);
19578            if old != *circ_error {
19579                d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19580            }
19581        }
19582
19583        self.graph.update_vertex_value_ref(vertex_id, circ_error);
19584        self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
19585    }
19586
19587    /// Dispatch point for one `ScheduleUnit::Cycle` (RFC #112, Stage 2).
19588    ///
19589    /// * `CycleDetection::Static` — today's behavior, byte-for-byte: stamp
19590    ///   `#CIRC!` on the (optionally dirty-filtered) members.
19591    /// * `CycleDetection::Runtime` — evaluate the SCC via
19592    ///   [`Self::evaluate_scc_unit`]. The recalc-plan dirty quirk maps to:
19593    ///   no dirty member → skip the task entirely (values stand); any dirty
19594    ///   member → the whole SCC evaluates (an SCC cannot be partially
19595    ///   evaluated).
19596    ///
19597    /// Returns the number of `#CIRC!`-stamped vertices, so call sites can
19598    /// keep their `cycle_errors` accounting (`> 0` ⇒ count the unit).
19599    fn handle_cycle_unit(
19600        &mut self,
19601        cycle: &[VertexId],
19602        mut delta: Option<&mut DeltaCollector>,
19603        dirty_filter: Option<&FxHashSet<VertexId>>,
19604        cancel_flag: Option<&AtomicBool>,
19605    ) -> Result<usize, ExcelError> {
19606        self.resource_checkpoint(cycle.len() as u64)?;
19607        match self.config.cycle.detection {
19608            CycleDetection::Static => {
19609                Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
19610            }
19611            CycleDetection::Runtime => {
19612                if let Some(filter) = dirty_filter
19613                    && !cycle.iter().any(|v| filter.contains(v))
19614                {
19615                    return Ok(0);
19616                }
19617                // Both policies share `evaluate_scc_unit`; they differ only
19618                // in the settle loop's live-cycle arm (Error stamps,
19619                // Iterate keeps passing — RFC #113).
19620                self.evaluate_scc_unit(cycle, delta, cancel_flag)
19621            }
19622        }
19623    }
19624
19625    /// Evaluate one statically-cyclic SCC under `CycleDetection::Runtime`
19626    /// (design doc `formualizer-stage2-scc-evaluation-design.md` §3; contract
19627    /// spec §3; Iterate policy arm per RFC #113).
19628    ///
19629    /// Phantom SCCs (live-acyclic) produce ordinary values under both
19630    /// policies; live cycles get `#CIRC!` with live-cycle-only blast radius
19631    /// under `CyclePolicy::Error`, or Excel-style iterative calculation
19632    /// (converge per spec §6 or cap at `max_iterations` passes) under
19633    /// `CyclePolicy::Iterate`. Runs sequentially on the
19634    /// coordinating thread; commits are write-through per member (no
19635    /// `ComputedWriteBuffer` — that buffer is scoped to layer evaluation and
19636    /// always flushed before a Cycle unit runs, G1), so later members' scalar
19637    /// *and* range reads observe earlier members' results through the overlay
19638    /// cascade. Deltas are recorded once per member at end of task (G11).
19639    ///
19640    /// Returns the number of vertices stamped `#CIRC!`.
19641    ///
19642    /// `pub(crate)` so tests can drive SCC shapes (e.g. name-vertex members)
19643    /// that ingest-time cycle rejection makes unreachable via public edits.
19644    pub(crate) fn evaluate_scc_unit(
19645        &mut self,
19646        cycle: &[VertexId],
19647        mut delta: Option<&mut DeltaCollector>,
19648        cancel_flag: Option<&AtomicBool>,
19649    ) -> Result<usize, ExcelError> {
19650        struct SccMember {
19651            vertex: VertexId,
19652            cell: Option<CellRef>,
19653        }
19654
19655        let task_start = crate::instant::FzInstant::now();
19656
19657        // ── 0. Member order (spec §7.13): cells ascending (sheet, row, col);
19658        // name vertices after, lexicographic by folded canonical name; any
19659        // other vertex kind (defensive — `get_evaluation_vertices` only emits
19660        // formula/name kinds) last by id, never evaluated.
19661        let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
19662        let mut name_members: Vec<(VertexId, String)> = Vec::new();
19663        let mut other_members: Vec<VertexId> = Vec::new();
19664        for &v in cycle {
19665            match self.graph.get_vertex_kind(v) {
19666                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
19667                    match self.graph.get_cell_ref(v) {
19668                        Some(cell) => cell_members.push((v, cell)),
19669                        None => other_members.push(v),
19670                    }
19671                }
19672                VertexKind::NamedScalar | VertexKind::NamedArray => {
19673                    match self.graph.name_key_for_vertex(v) {
19674                        Some(key) => name_members.push((v, key)),
19675                        None => other_members.push(v),
19676                    }
19677                }
19678                _ => other_members.push(v),
19679            }
19680        }
19681        cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
19682        name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
19683        other_members.sort_unstable();
19684
19685        let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
19686        let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
19687        let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
19688        for (v, c) in &cell_members {
19689            members.push(SccMember {
19690                vertex: *v,
19691                cell: Some(*c),
19692            });
19693        }
19694        for (v, _) in &name_members {
19695            members.push(SccMember {
19696                vertex: *v,
19697                cell: None,
19698            });
19699        }
19700        for v in &other_members {
19701            members.push(SccMember {
19702                vertex: *v,
19703                cell: None,
19704            });
19705        }
19706        let n = members.len();
19707        // Indices addressable by the collector (cells + names); `other`
19708        // members can be neither edge sources nor targets.
19709        let recordable = cell_refs.len() + name_keys.len();
19710
19711        let circ_error = LiteralValue::Error(
19712            ExcelError::new(ExcelErrorKind::Circ)
19713                .with_message("Circular dependency detected".to_string()),
19714        );
19715
19716        // ── 0b. Spec-§4 persistence repair: structural edits clear computed
19717        // overlays wholesale (`clear_computed_overlay_after_row/_col`), but
19718        // an iterating member's committed value is cycle STATE, not a
19719        // recomputable cache — and in canonical mode the overlay is its ONLY
19720        // home. If the overlay entry vanished since the last recalc, re-seed
19721        // it from the end-of-recalc snapshot (`iterative_state_values`) so
19722        // pass-1 reads (scalar AND range, via the overlay cascade) observe
19723        // the persisted value instead of silently restarting at Empty→0.
19724        // (Found by the iterate edge corpus: inserting/deleting an unrelated
19725        // row reset accumulators, violating spec §4/§7.15.)
19726        if !self.iterative_state_values.is_empty() {
19727            let restore: Vec<(VertexId, LiteralValue)> = members
19728                .iter()
19729                .filter_map(|m| {
19730                    let cell = m.cell?;
19731                    let persisted = self.iterative_state_values.get(&m.vertex)?;
19732                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
19733                    let overlay = self
19734                        .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19735                        .unwrap_or(LiteralValue::Empty);
19736                    if matches!(overlay, LiteralValue::Empty) {
19737                        Some((m.vertex, persisted.clone()))
19738                    } else {
19739                        None
19740                    }
19741                })
19742                .collect();
19743            for (vertex, value) in restore {
19744                self.mirror_vertex_value_to_overlay(vertex, &value);
19745            }
19746        }
19747
19748        // ── 1. Pre-task value snapshot (overlay-first for cells — G3; the
19749        // graph value map may be evicted in value-cache-disabled mode).
19750        let snapshot: Vec<LiteralValue> = members
19751            .iter()
19752            .map(|m| match m.cell {
19753                Some(cell) => {
19754                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
19755                    self.read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19756                        .unwrap_or(LiteralValue::Empty)
19757                }
19758                None => self
19759                    .graph
19760                    .get_value(m.vertex)
19761                    .unwrap_or(LiteralValue::Empty),
19762            })
19763            .collect();
19764
19765        // ── 2. Pre-scan: spill anchors (FormulaArray) are stamped `#CIRC!`
19766        // with full spill teardown (spec §7.9, #115) and excluded from
19767        // evaluation. They stay recordable edge TARGETS (readers see `#CIRC!`
19768        // and propagate). Non-evaluable defensive members are excluded too.
19769        let mut excluded = vec![false; n];
19770        let mut last_value = snapshot.clone();
19771        let mut stamped = 0usize;
19772        for (i, m) in members.iter().enumerate() {
19773            match self.graph.get_vertex_kind(m.vertex) {
19774                VertexKind::FormulaArray => {
19775                    // Deltas for the cleared spill-region cells (non-members)
19776                    // can only be recorded here; the anchor's own delta is
19777                    // covered by the end-of-task snapshot comparison (dedup).
19778                    self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
19779                    excluded[i] = true;
19780                    last_value[i] = circ_error.clone();
19781                    stamped += 1;
19782                }
19783                VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
19784                _ => excluded[i] = true,
19785            }
19786        }
19787
19788        let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
19789
19790        // Per-member live out-edges, refreshed whenever a member re-runs.
19791        let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
19792        // Position of each member in the most recent pass (-1 = did not run).
19793        let mut pos: Vec<i64> = vec![-1; n];
19794        // Whether each member's committed value changed in the most recent pass.
19795        let mut changed = vec![false; n];
19796
19797        // Evaluate-and-commit one member; returns Ok(true) when the member was
19798        // stamped `#CIRC!` (array result — would-be spill anchor, spec §7.9).
19799        macro_rules! run_member {
19800            ($i:expr) => {{
19801                let i: usize = $i;
19802                let m = &members[i];
19803                if i < recordable {
19804                    collector.set_current(i as u32);
19805                }
19806                let value = {
19807                    let ctx = RecordingContext::new(&*self, &collector);
19808                    match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
19809                        Ok(v) => v,
19810                        Err(e) => LiteralValue::Error(e),
19811                    }
19812                };
19813                let is_cell_formula = m.cell.is_some();
19814                if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
19815                    // A member that *would* spill inside an SCC gets the
19816                    // conservative §7.9 verdict. It has never spilled before
19817                    // (a prior spill would make it FormulaArray, pre-stamped
19818                    // above), so there is no projection to tear down.
19819                    self.stamp_cycle_error(m.vertex, &circ_error, None);
19820                    excluded[i] = true;
19821                    stamped += 1;
19822                    changed[i] = last_value[i] != circ_error;
19823                    last_value[i] = circ_error.clone();
19824                } else {
19825                    self.graph.update_vertex_value_ref(m.vertex, &value);
19826                    self.mirror_vertex_value_to_overlay(m.vertex, &value);
19827                    // §7.14 invariant (G2): a formula member must never be
19828                    // shadowed by a user/delta overlay entry, or iteration
19829                    // reads would silently diverge from committed values.
19830                    #[cfg(debug_assertions)]
19831                    if let Some(cell) = m.cell {
19832                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19833                        debug_assert!(
19834                            self.read_delta_overlay_cell(
19835                                &sheet_name,
19836                                cell.coord.row() + 1,
19837                                cell.coord.col() + 1
19838                            )
19839                            .is_none(),
19840                            "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
19841                            cell.coord.row() + 1,
19842                            cell.coord.col() + 1
19843                        );
19844                    }
19845                    changed[i] = last_value[i] != value;
19846                    last_value[i] = value;
19847                }
19848            }};
19849        }
19850
19851        let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
19852            if let Some(flag) = flag
19853                && flag.load(Ordering::Relaxed)
19854            {
19855                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
19856                    .with_message("Evaluation cancelled during SCC evaluation".to_string()));
19857            }
19858            Ok(())
19859        };
19860
19861        // ── 3. Pass 1: all evaluable members in member order.
19862        check_cancel(cancel_flag)?;
19863        let mut passes = 1usize;
19864        {
19865            let mut p = 0i64;
19866            for i in 0..n {
19867                if excluded[i] {
19868                    continue;
19869                }
19870                run_member!(i);
19871                pos[i] = p;
19872                p += 1;
19873            }
19874        }
19875
19876        // ── 4. Settle loop (design doc §3 step 4; RFC #113 policy arm).
19877        //
19878        // Acyclic classifications settle stale readers exactly (identical
19879        // under both policies — phantom SCCs never iterate). A witnessed
19880        // live cycle dispatches on policy: `Error` stamps `#CIRC!` and
19881        // stops; `Iterate` keeps running full passes over all members in
19882        // member order until converged (spec §6) or capped at
19883        // `max_iterations` total passes. A live cycle that only appears
19884        // mid-settle takes the same arm, and a cycle that dissolves
19885        // mid-iteration falls back to exact acyclic settling.
19886        //
19887        // Defensive acyclic budget: the acyclic settle is monotone, so more
19888        // than |SCC| + 2 settle passes can only be a bug; cap hits stamp the
19889        // remainder and set telemetry. Tracked via `settle_passes` so
19890        // iteration passes (legitimately many) don't consume the budget.
19891        let policy = self.config.cycle.policy;
19892        let cap = n + 2;
19893        let mut witnessed_cycles = 0usize;
19894        let mut capped = false;
19895        // ── Iterate-policy state ──
19896        let mut iterating = false;
19897        let mut converged = false;
19898        let mut exact_fixed_point = false;
19899        // Values committed by the last *full* pass; `None` until the first
19900        // iteration pass runs (pass 1 has no predecessor to compare against)
19901        // and reset when a settle pass runs (no cross-kind comparisons).
19902        let mut prev_pass: Option<Vec<LiteralValue>> = None;
19903        // Final-round convergence stats (overwritten per round so the values
19904        // reported are the ones observed at stop).
19905        let mut iter_max_delta = 0f64;
19906        let mut iter_nan_converged = 0usize;
19907        // Acyclic stale-reader re-eval passes (defensive budget; under pure
19908        // Error flow `1 + settle_passes == passes`, preserving Stage-2
19909        // behavior exactly).
19910        let mut settle_passes = 0usize;
19911        loop {
19912            // Drain this pass's recordings; members that ran replace their
19913            // out-edge set, members that didn't keep last-known edges.
19914            let drained = collector.take_edges();
19915            for i in 0..n {
19916                if pos[i] >= 0 {
19917                    out_edges[i].clear();
19918                }
19919            }
19920            for (from, to) in drained {
19921                debug_assert!(
19922                    pos[from as usize] >= 0,
19923                    "edge from a member that did not run"
19924                );
19925                out_edges[from as usize].push(to);
19926            }
19927            let mut edges: Vec<(u32, u32)> = Vec::new();
19928            for (i, outs) in out_edges.iter().enumerate() {
19929                if excluded[i] {
19930                    continue;
19931                }
19932                for &t in outs {
19933                    edges.push((i as u32, t));
19934                }
19935            }
19936            edges.sort_unstable();
19937            edges.dedup();
19938
19939            let analysis = analyze_live_graph(n, &edges);
19940
19941            if analysis.cycle_count > 0 {
19942                // Classification repeats every iteration pass under
19943                // `Iterate`; record the widest single witness instead of
19944                // accumulating so the count stays "distinct live cycles".
19945                witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
19946                match policy {
19947                    CyclePolicy::Error => {
19948                        // POLICY (Error): stamp every member of a live cycle,
19949                        // then one settling pass over the remaining members in
19950                        // live-topological order so error propagation
19951                        // downstream is consistent (spec §3.4). Blast radius =
19952                        // live cycles only.
19953                        for i in 0..n {
19954                            if analysis.in_cycle[i] && !excluded[i] {
19955                                self.stamp_cycle_error(members[i].vertex, &circ_error, None);
19956                                excluded[i] = true;
19957                                last_value[i] = circ_error.clone();
19958                                stamped += 1;
19959                            }
19960                        }
19961                        check_cancel(cancel_flag)?;
19962                        let order: Vec<usize> = analysis
19963                            .topo
19964                            .iter()
19965                            .map(|&i| i as usize)
19966                            .filter(|&i| !excluded[i])
19967                            .collect();
19968                        if !order.is_empty() {
19969                            passes += 1;
19970                            for i in order {
19971                                run_member!(i);
19972                            }
19973                        }
19974                        break;
19975                    }
19976                    CyclePolicy::Iterate {
19977                        max_iterations,
19978                        max_change,
19979                    } => {
19980                        // POLICY (Iterate), spec §3.5/§6.
19981                        iterating = true;
19982
19983                        // Convergence test: the full pass that just completed
19984                        // vs the previous full pass, per the spec-§6 rules.
19985                        // `prev_pass` is `None` until an iteration pass has
19986                        // run — pass 1 has no predecessor, so no convergence
19987                        // test occurs before the second pass (spec §6).
19988                        if let Some(prev) = &prev_pass {
19989                            let mut round_max_delta = 0f64;
19990                            let mut round_nan = 0usize;
19991                            let mut all_converged = true;
19992                            let mut round_exact = true;
19993                            for i in 0..n {
19994                                if excluded[i] {
19995                                    // Stamped mid-iteration (array result,
19996                                    // §7.9): the value is pinned and cannot
19997                                    // change again — trivially settled.
19998                                    continue;
19999                                }
20000                                let out = crate::engine::convergence::values_converged(
20001                                    &prev[i],
20002                                    &last_value[i],
20003                                    max_change,
20004                                    self.config.date_system,
20005                                );
20006                                if out.nan_converged {
20007                                    round_nan += 1;
20008                                }
20009                                if let Some(d) = out.abs_delta {
20010                                    round_max_delta = round_max_delta.max(d);
20011                                    if d != 0.0 {
20012                                        round_exact = false;
20013                                    }
20014                                }
20015                                if !out.converged {
20016                                    all_converged = false;
20017                                }
20018                            }
20019                            // Overwrite (not max): telemetry reports the
20020                            // round observed at stop.
20021                            iter_max_delta = round_max_delta;
20022                            iter_nan_converged = round_nan;
20023                            if all_converged {
20024                                exact_fixed_point = round_exact;
20025                                converged = true;
20026                                break;
20027                            }
20028                        }
20029
20030                        // ── Pass-counting reconciliation (spec §6/§7.6):
20031                        // `max_iterations` counts TOTAL passes, pass 1
20032                        // included, and pass 1 has already run by the time a
20033                        // live cycle is first witnessed here. The budget is
20034                        // therefore checked BEFORE evaluating anything more:
20035                        // with `max_iterations: 1` we stop right here — each
20036                        // member was evaluated exactly once this recalc (the
20037                        // Excel accumulator contract) and no convergence test
20038                        // ran (`prev_pass` is still `None`). Capping keeps
20039                        // the last committed values and is NOT an error
20040                        // (Excel parity); telemetry records it.
20041                        if passes >= max_iterations as usize {
20042                            capped = true;
20043                            break;
20044                        }
20045
20046                        check_cancel(cancel_flag)?;
20047                        // One more full pass over every evaluable member in
20048                        // member order (Gauss–Seidel: each commit is visible
20049                        // to later members within the pass). Live edges
20050                        // re-record — guards can flip near convergence
20051                        // (§7.3) — so classification repeats next time
20052                        // around, and a cycle that dissolves drops back to
20053                        // the exact acyclic settle below.
20054                        prev_pass = Some(last_value.clone());
20055                        for x in pos.iter_mut() {
20056                            *x = -1;
20057                        }
20058                        changed.fill(false);
20059                        passes += 1;
20060                        let mut p = 0i64;
20061                        for i in 0..n {
20062                            if excluded[i] {
20063                                continue;
20064                            }
20065                            run_member!(i);
20066                            pos[i] = p;
20067                            p += 1;
20068                        }
20069                        continue;
20070                    }
20071                }
20072            }
20073
20074            // Acyclic: find stale readers — members whose live read of `to`
20075            // happened before `to`'s value changed in the pass that just ran.
20076            let mut stale: Vec<usize> = Vec::new();
20077            for i in 0..n {
20078                if excluded[i] {
20079                    continue;
20080                }
20081                let is_stale = out_edges[i].iter().any(|&t| {
20082                    let t = t as usize;
20083                    changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
20084                });
20085                if is_stale {
20086                    stale.push(i);
20087                }
20088            }
20089            if stale.is_empty() {
20090                break; // values exact — phantom SCC (or dissolved live cycle)
20091            }
20092            if 1 + settle_passes >= cap {
20093                // Defensive only; hitting this is a bug (loud telemetry).
20094                capped = true;
20095                for (i, m) in members.iter().enumerate() {
20096                    if !excluded[i] {
20097                        self.stamp_cycle_error(m.vertex, &circ_error, None);
20098                        excluded[i] = true;
20099                        last_value[i] = circ_error.clone();
20100                        stamped += 1;
20101                    }
20102                }
20103                break;
20104            }
20105
20106            check_cancel(cancel_flag)?;
20107            // Re-evaluate stale readers in live-topo order, recording fresh
20108            // edges (branches may flip on re-eval — spec §7.3 — which is why
20109            // classification repeats).
20110            // A settle pass is a partial sweep: drop the full-pass baseline
20111            // so a live cycle (re)appearing afterwards never compares values
20112            // across mixed pass kinds.
20113            prev_pass = None;
20114            let topo_pos = analysis.topo_positions();
20115            stale.sort_unstable_by_key(|&i| topo_pos[i]);
20116            for x in pos.iter_mut() {
20117                *x = -1;
20118            }
20119            changed.fill(false);
20120            passes += 1;
20121            settle_passes += 1;
20122            for (p, i) in stale.into_iter().enumerate() {
20123                run_member!(i);
20124                pos[i] = p as i64;
20125            }
20126        }
20127
20128        // Post-work boundary: members are committed write-through, so a
20129        // member that ran across a live cancellation is already visible; the
20130        // task still must not complete (no delta, retention or iteration
20131        // state). The members stay dirty for the retry, as with the
20132        // mid-task checks above.
20133        self.live_cancellation_after_work(SCC_CANCELLED)?;
20134
20135        // Iteration that ended because the live cycle dissolved and the
20136        // acyclic settle reached exactness counts as converged (values are
20137        // exact, strictly better than threshold-converged). The defensive
20138        // settle cap (`capped` + stamping) is not.
20139        if iterating && !converged && !capped {
20140            converged = true;
20141        }
20142
20143        // ── 5. End of task: one delta per member whose final value differs
20144        // from the pre-task snapshot (spec §3 side-effect rule, G11).
20145        collector.clear_current();
20146        if let Some(d) = delta
20147            && d.mode != DeltaMode::Off
20148        {
20149            for (i, m) in members.iter().enumerate() {
20150                if let Some(cell) = m.cell
20151                    && last_value[i] != snapshot[i]
20152                {
20153                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20154                }
20155            }
20156        }
20157
20158        // Members of an SCC that iterated re-evaluate on EVERY recalc, like
20159        // Excel's circular cells: register them for the end-of-recalc
20160        // volatile-like redirty (see `pending_iterative_redirty`). Marking
20161        // any one member propagates around the (strongly connected) SCC and
20162        // to downstream dependents, but all members are registered so the
20163        // contract survives partial structural edits between recalcs.
20164        //
20165        // Exception (#368): an SCC that stopped on an
20166        // exact fixed point — every member reproduced its previous value
20167        // bit-for-bit before the pass cap, no NaN identity, no volatile or
20168        // dynamic-reference member — cannot change on a re-run with the same
20169        // inputs, so it is retained clean. The dirty graph decides when it
20170        // runs again. Tolerance-only convergence (|Δ| < max_change but ≠ 0),
20171        // capped SCCs (including the `max_iterations: 1` accumulator
20172        // contract) and volatile cycles keep the per-recalc redirty.
20173        // Whatever the outcome, this task supersedes any earlier retention
20174        // of its members.
20175        if !self.retained_scc_members.is_empty() {
20176            for m in members.iter() {
20177                self.retained_scc_members.remove(&m.vertex);
20178            }
20179        }
20180        if iterating {
20181            let retain = converged
20182                && !capped
20183                && exact_fixed_point
20184                && iter_nan_converged == 0
20185                && members
20186                    .iter()
20187                    .all(|m| !self.graph.is_volatile(m.vertex) && !self.graph.is_dynamic(m.vertex));
20188            if retain {
20189                if self.retained_scc_members.is_empty() {
20190                    self.retained_scc_config_fingerprint = self.retained_scc_config_fingerprint();
20191                    self.retained_scc_function_epoch_seen =
20192                        crate::function_registry::semantic_epoch();
20193                    self.retained_scc_provider_revision_seen =
20194                        self.resolver.planning_semantic_revision();
20195                }
20196                let scc_id = self.next_retained_scc_id;
20197                self.next_retained_scc_id = self.next_retained_scc_id.wrapping_add(1);
20198                for (i, m) in members.iter().enumerate() {
20199                    self.retained_scc_members.insert(m.vertex, scc_id);
20200                    // §4 persistence snapshot, written once: retained
20201                    // members do not pass through `redirty_for_next_recalc`.
20202                    if matches!(last_value[i], LiteralValue::Empty) {
20203                        self.iterative_state_values.remove(&m.vertex);
20204                    } else {
20205                        self.iterative_state_values
20206                            .insert(m.vertex, last_value[i].clone());
20207                    }
20208                }
20209            } else {
20210                self.pending_iterative_redirty
20211                    .extend(members.iter().map(|m| m.vertex));
20212            }
20213        } else if !self.iterative_state_values.is_empty() {
20214            // The cycle dissolved (phantom settle or `#CIRC!` stamping):
20215            // these members are ordinary formulas again and must not carry
20216            // stale iteration state into a future cycle.
20217            for m in members.iter() {
20218                self.iterative_state_values.remove(&m.vertex);
20219            }
20220        }
20221
20222        {
20223            let t = &mut self.last_cycle_telemetry;
20224            t.static_sccs += 1;
20225            if witnessed_cycles == 0 && stamped == 0 && !capped {
20226                t.phantom_sccs += 1;
20227            }
20228            t.live_cycles_witnessed += witnessed_cycles;
20229            t.circ_cells_stamped += stamped;
20230            t.settle_passes_total += passes;
20231            t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
20232            if iterating {
20233                t.iterated_sccs += 1;
20234                if converged {
20235                    t.converged_sccs += 1;
20236                }
20237                t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
20238                t.nan_converged += iter_nan_converged;
20239            }
20240            if capped {
20241                t.capped_sccs += 1;
20242            }
20243            t.elapsed_ms += task_start.elapsed().as_millis();
20244        }
20245
20246        Ok(stamped)
20247    }
20248
20249    /// Recorded sibling of [`Self::evaluate_vertex_immutable`]: evaluates one
20250    /// SCC member's AST via an [`Interpreter`] over a [`RecordingContext`] so
20251    /// reads that actually occur are captured as live edges. Value semantics
20252    /// must match `evaluate_vertex_immutable` exactly (including the missing-
20253    /// AST `Number(0.0)` quirk, G14); named Cell/Range/Literal definitions
20254    /// delegate to it after recording the definition region by hand (those
20255    /// reads bypass the context).
20256    fn evaluate_vertex_recorded(
20257        &self,
20258        vertex_id: VertexId,
20259        ctx: &RecordingContext<'_, R>,
20260        collector: &LiveEdgeCollector,
20261    ) -> Result<LiteralValue, ExcelError> {
20262        if !self.graph.vertex_exists(vertex_id) {
20263            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
20264                .with_message(format!("Vertex not found: {vertex_id:?}")));
20265        }
20266
20267        let kind = self.graph.get_vertex_kind(vertex_id);
20268        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
20269
20270        match kind {
20271            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20272                let Some(view) = self.graph.formula_view(vertex_id) else {
20273                    return Ok(LiteralValue::Number(0.0)); // G14 quirk
20274                };
20275                let sheet_name = self.graph.sheet_name(sheet_id);
20276                let cell_ref = self
20277                    .graph
20278                    .get_cell_ref(vertex_id)
20279                    .expect("cell ref for vertex");
20280                let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref)
20281                    .with_resolved_functions(&self.resolved_functions);
20282                interpreter
20283                    .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
20284                    .map(|cv| {
20285                        let format = cv.format_id();
20286                        self.record_derived_format(vertex_id, format);
20287                        self.materialize_formula_result(vertex_id, cv)
20288                    })
20289                    .or_else(|error| self.fit_formula_error(vertex_id, error))
20290            }
20291            VertexKind::NamedScalar | VertexKind::NamedArray => {
20292                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
20293                    ExcelError::new(ExcelErrorKind::Name)
20294                        .with_message("Named range metadata missing".to_string())
20295                })?;
20296
20297                match &named_range.definition {
20298                    NamedDefinition::Formula { ast, .. } => {
20299                        let context_sheet = match named_range.scope {
20300                            NameScope::Sheet(id) => id,
20301                            NameScope::Workbook => sheet_id,
20302                        };
20303                        let sheet_name = self.graph.sheet_name(context_sheet);
20304                        let cell_ref = self
20305                            .graph
20306                            .get_cell_ref(vertex_id)
20307                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
20308                        let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref)
20309                            .with_resolved_functions(&self.resolved_functions);
20310                        if kind == VertexKind::NamedScalar {
20311                            interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
20312                        } else {
20313                            match interpreter.evaluate_ast(ast) {
20314                                Ok(cv) => match cv.into_literal() {
20315                                    v @ LiteralValue::Array(_) => Ok(v),
20316                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
20317                                },
20318                                Err(err) => Ok(LiteralValue::Error(err)),
20319                            }
20320                        }
20321                    }
20322                    NamedDefinition::Cell(cell_ref) => {
20323                        // The definition is read via direct grid access in
20324                        // `evaluate_vertex_immutable`; record the live edge
20325                        // by hand before delegating.
20326                        collector.record_scalar(
20327                            cell_ref.sheet_id,
20328                            cell_ref.coord.row(),
20329                            cell_ref.coord.col(),
20330                        );
20331                        self.evaluate_vertex_immutable(vertex_id)
20332                    }
20333                    NamedDefinition::Range(range_ref) => {
20334                        if range_ref.start.sheet_id == range_ref.end.sheet_id {
20335                            collector.record_rect(
20336                                range_ref.start.sheet_id,
20337                                range_ref.start.coord.row(),
20338                                range_ref.start.coord.col(),
20339                                range_ref.end.coord.row(),
20340                                range_ref.end.coord.col(),
20341                            );
20342                        }
20343                        self.evaluate_vertex_immutable(vertex_id)
20344                    }
20345                    NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
20346                }
20347            }
20348            _ => self.evaluate_vertex_immutable(vertex_id),
20349        }
20350    }
20351
20352    /// Pending source occupancy is independent of formula preparation and value caches.
20353    fn pending_spill_occupied(&self, anchor: CellRef, end_row: u32, end_col: u32) -> bool {
20354        let sheet = self.graph.sheet_name(anchor.sheet_id);
20355        let package = self
20356            .staged_formulas
20357            .get(sheet)
20358            .and_then(|staged| staged.deferred_package.as_ref());
20359        self.staged_formula_index.occupies_spill(
20360            sheet,
20361            (anchor.coord.row() + 1, anchor.coord.col() + 1),
20362            (end_row + 1, end_col + 1),
20363            |point| package.is_some_and(|package| package.suppressed.contains(&point)),
20364        )
20365    }
20366
20367    fn remember_pending_spill(
20368        &mut self,
20369        vertex: VertexId,
20370        anchor: CellRef,
20371        region: Region,
20372    ) -> Result<(), ExcelError> {
20373        self.cancellation_checkpoint("pending spill occupancy")?;
20374        self.resource_checkpoint(1)?;
20375        if let Some(entry) = self
20376            .blocked_pending_spills
20377            .iter_mut()
20378            .find(|entry| entry.0 == vertex)
20379        {
20380            *entry = (vertex, anchor, region);
20381            return Ok(());
20382        }
20383        if self.blocked_pending_spills.len() == self.blocked_pending_spills.capacity() {
20384            // Geometric growth avoids copying every existing retry entry for
20385            // every new blocked anchor. Admit the entire capacity increment.
20386            let additional = self.blocked_pending_spills.capacity().max(1);
20387            let bytes =
20388                (additional as u64)
20389                    .saturating_mul(std::mem::size_of::<(VertexId, CellRef, Region)>() as u64);
20390            if let Some(ledger) = self.active_resource_ledger.as_mut() {
20391                ledger
20392                    .reserve_retained(bytes)
20393                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20394                self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
20395            }
20396            if self
20397                .blocked_pending_spills
20398                .try_reserve_exact(additional)
20399                .is_err()
20400            {
20401                if let Some(ledger) = self.active_resource_ledger.as_mut() {
20402                    ledger
20403                        .release_retained(bytes)
20404                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20405                    self.source_cache_accounted -= bytes;
20406                }
20407                return Err(crate::engine::ResourceLedgerError::Exhausted(
20408                    formualizer_common::ResourceExhaustionDetail {
20409                        reason: formualizer_common::ResourceExhaustionReason::RetainedMemory,
20410                        limit: u64::MAX,
20411                        observed: bytes,
20412                        request_id: None,
20413                    },
20414                )
20415                .into_excel_error());
20416            }
20417        }
20418        self.blocked_pending_spills.push((vertex, anchor, region));
20419        Ok(())
20420    }
20421
20422    // Successful edits wake only intersecting attempted regions. Keep the entry
20423    // until evaluation: logged edits may still roll back, and materializing a
20424    // pending formula must not destroy its anchor's occupancy retry information.
20425    fn invalidate_pending_spills(&mut self, scope: StructuralScope) {
20426        if let StructuralScope::RemovedSheet(sheet) = scope {
20427            self.blocked_pending_spills
20428                .retain(|entry| entry.1.sheet_id != sheet);
20429            return;
20430        }
20431        for &(vertex, anchor, region) in &self.blocked_pending_spills {
20432            let affected = match scope {
20433                StructuralScope::Cell { sheet, row, col } => {
20434                    region.intersects(&Region::point(sheet, row, col))
20435                }
20436                StructuralScope::Region(changed) => region.intersects(&changed),
20437                StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => {
20438                    region.sheet_id() == sheet
20439                }
20440                StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => true,
20441            };
20442            if affected
20443                && self.graph.vertex_exists(vertex)
20444                && self.graph.get_cell_ref(vertex) == Some(anchor)
20445                && matches!(
20446                    self.graph.get_vertex_kind(vertex),
20447                    VertexKind::FormulaScalar | VertexKind::FormulaArray
20448                )
20449            {
20450                self.graph.mark_vertex_dirty(vertex);
20451            }
20452        }
20453    }
20454
20455    fn guard_pending_spill_commit(
20456        &mut self,
20457        anchor_vertex: VertexId,
20458        targets: &[CellRef],
20459    ) -> Result<(), ExcelError> {
20460        let Some(anchor) = self.graph.get_cell_ref(anchor_vertex) else {
20461            return Ok(());
20462        };
20463        let Some(last) = targets.last() else {
20464            return Ok(());
20465        };
20466        let occupied = self.pending_spill_occupied(anchor, last.coord.row(), last.coord.col())
20467            || self
20468                .graph
20469                .table_intersects_spill(anchor, last.coord.row(), last.coord.col());
20470        if (occupied
20471            || self
20472                .blocked_pending_spills
20473                .iter()
20474                .any(|entry| entry.0 == anchor_vertex))
20475            && let Err(error) = self.remember_pending_spill(
20476                anchor_vertex,
20477                anchor,
20478                Region::rect(
20479                    anchor.sheet_id,
20480                    anchor.coord.row(),
20481                    last.coord.row(),
20482                    anchor.coord.col(),
20483                    last.coord.col(),
20484                ),
20485            )
20486        {
20487            self.spill_mgr.release_owner(anchor_vertex);
20488            return Err(error);
20489        }
20490        if occupied {
20491            self.spill_mgr.release_owner(anchor_vertex);
20492            return Err(ExcelError::new(ExcelErrorKind::Spill)
20493                .with_message("Spill blocked")
20494                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20495                    expected_rows: last.coord.row() - anchor.coord.row() + 1,
20496                    expected_cols: last.coord.col() - anchor.coord.col() + 1,
20497                }));
20498        }
20499        Ok(())
20500    }
20501
20502    /// Commit spill via shim and mirror resulting cells into Arrow overlay.
20503    fn commit_spill_and_mirror(
20504        &mut self,
20505        anchor_vertex: VertexId,
20506        targets: &[CellRef],
20507        rows: Vec<Vec<LiteralValue>>,
20508        delta: Option<&mut DeltaCollector>,
20509    ) -> Result<(), ExcelError> {
20510        self.guard_pending_spill_commit(anchor_vertex, targets)?;
20511        let prev_spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
20512
20513        if let Some(delta) = delta
20514            && delta.mode != DeltaMode::Off
20515        {
20516            let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20517                targets.iter().copied().collect();
20518            let empty = LiteralValue::Empty;
20519
20520            // Clears (prev - targets)
20521            for cell in prev_spill_cells.iter() {
20522                if target_set.contains(cell) {
20523                    continue;
20524                }
20525                let sheet_name = self.graph.sheet_name(cell.sheet_id);
20526                let old = self
20527                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20528                    .unwrap_or(LiteralValue::Empty);
20529                if old != empty {
20530                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20531                }
20532            }
20533
20534            // Writes (targets)
20535            if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
20536                let width = rows[0].len();
20537                for (idx, cell) in targets.iter().enumerate() {
20538                    let r_off = idx / width;
20539                    let c_off = idx % width;
20540                    let new = rows
20541                        .get(r_off)
20542                        .and_then(|r| r.get(c_off))
20543                        .cloned()
20544                        .unwrap_or(LiteralValue::Empty);
20545                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
20546                    let old = self
20547                        .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20548                        .unwrap_or(LiteralValue::Empty);
20549                    if old != new {
20550                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20551                    }
20552                }
20553            } else {
20554                // Degenerate shapes: if we have targets but no rows, treat as writing Empty.
20555                for cell in targets.iter() {
20556                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
20557                    let old = self
20558                        .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20559                        .unwrap_or(LiteralValue::Empty);
20560                    if !matches!(old, LiteralValue::Empty) {
20561                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20562                    }
20563                }
20564            }
20565        }
20566
20567        // Commit via shim (releases locks). When the graph value cache is disabled (Arrow-canonical
20568        // values), plan/commit must consult Arrow storage to detect non-empty value blockers.
20569        let arrow_sheets = &self.arrow_sheets;
20570        self.spill_mgr.commit_array_with_value_probe(
20571            &mut self.graph,
20572            anchor_vertex,
20573            targets,
20574            rows.clone(),
20575            |g, cell| {
20576                let sheet_name = g.sheet_name(cell.sheet_id);
20577                let asheet = arrow_sheets.sheet(sheet_name)?;
20578                let r0 = cell.coord.row() as usize;
20579                let c0 = cell.coord.col() as usize;
20580                let v = asheet.get_cell_value(r0, c0);
20581                if matches!(v, LiteralValue::Empty) {
20582                    None
20583                } else {
20584                    Some(v)
20585                }
20586            },
20587        )?;
20588
20589        self.blocked_pending_spills
20590            .retain(|entry| entry.0 != anchor_vertex);
20591        self.spill_blocker_waiters.forget(anchor_vertex);
20592        if let Some(scope) = Self::structural_scope_from_cells(&prev_spill_cells) {
20593            self.record_structural_change(scope);
20594        }
20595        if let Some(scope) = Self::structural_scope_from_cells(targets) {
20596            self.record_structural_change(scope);
20597        }
20598
20599        if self.config.arrow_storage_enabled
20600            && self.config.delta_overlay_enabled
20601            && self.config.write_formula_overlay_enabled
20602        {
20603            if !prev_spill_cells.is_empty() {
20604                let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20605                    targets.iter().copied().collect();
20606                let empty = LiteralValue::Empty;
20607                for cell in prev_spill_cells.iter() {
20608                    if !target_set.contains(cell) {
20609                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20610                        self.mirror_value_to_computed_overlay(
20611                            &sheet_name,
20612                            cell.coord.row() + 1,
20613                            cell.coord.col() + 1,
20614                            &empty,
20615                        );
20616                    }
20617                }
20618            }
20619
20620            for (idx, cell) in targets.iter().enumerate() {
20621                if rows.is_empty() || rows[0].is_empty() {
20622                    break;
20623                }
20624                let width = rows[0].len();
20625                let r_off = idx / width;
20626                let c_off = idx % width;
20627                let v = rows[r_off][c_off].clone();
20628                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20629                self.mirror_value_to_computed_overlay(
20630                    &sheet_name,
20631                    cell.coord.row() + 1,
20632                    cell.coord.col() + 1,
20633                    &v,
20634                );
20635            }
20636        }
20637        Ok(())
20638    }
20639}
20640
20641#[cfg(test)]
20642#[path = "tests/authority_schedule_execution.rs"]
20643mod authority_schedule_execution;
20644
20645#[cfg(test)]
20646#[path = "tests/scalar_commit.rs"]
20647mod scalar_commit_tests;
20648
20649#[cfg(test)]
20650#[path = "tests/pending_spill.rs"]
20651mod pending_spill_tests;
20652
20653#[cfg(test)]
20654#[path = "tests/spill_batch_abort_192.rs"]
20655mod spill_batch_abort_192;
20656
20657// ── Effects pipeline (ticket 603) ──────────────────────────────────────────
20658//
20659// Compute → Plan → Apply separation for evaluation side-effects.
20660
20661use crate::engine::effects::Effect;
20662
20663// Scalar writes and spill clears stay inline; larger spill plans grow normally.
20664type PlannedEffects = smallvec::SmallVec<[Effect; 2]>;
20665use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
20666
20667impl<R> Engine<R>
20668where
20669    R: EvaluationContext,
20670{
20671    /// Plan effects for a single vertex after its value has been computed.
20672    ///
20673    /// This reads graph state but only performs lightweight mutations
20674    /// (`set_kind`, `spill_mgr.reserve`) that are needed for correctness
20675    /// during the planning phase.  Value-changing mutations are deferred to
20676    /// `apply_effect`.
20677    pub(crate) fn plan_vertex_effects(
20678        &mut self,
20679        vertex_id: VertexId,
20680        computed_value: LiteralValue,
20681    ) -> Result<PlannedEffects, ExcelError> {
20682        // FR3: a stale dynamic reader publishes nothing; FR2: anything else
20683        // leaves the dirty set at its commit (design §8.2).
20684        if self.freshness_armed() {
20685            if self.freshness_drop_stale(vertex_id) {
20686                return Ok(PlannedEffects::new());
20687            }
20688            let effects = self.plan_vertex_effects_unrecorded(vertex_id, computed_value)?;
20689            self.freshness_mark_committed(vertex_id);
20690            return Ok(effects);
20691        }
20692        self.plan_vertex_effects_unrecorded(vertex_id, computed_value)
20693    }
20694
20695    fn plan_vertex_effects_unrecorded(
20696        &mut self,
20697        vertex_id: VertexId,
20698        computed_value: LiteralValue,
20699    ) -> Result<PlannedEffects, ExcelError> {
20700        let kind = self.graph.get_vertex_kind(vertex_id);
20701        let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
20702
20703        // If this vertex's cell is currently covered by a spill from a different
20704        // anchor, ignore the computed result.  Formula vertices are exempt:
20705        // they must still evaluate so that overlapping spills produce #SPILL!.
20706        if !is_formula {
20707            if let Some(cell) = self.graph.get_cell_ref(vertex_id)
20708                && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
20709                && owner != vertex_id
20710            {
20711                return Ok(PlannedEffects::new());
20712            }
20713            // Non-formula vertices: store value as-is (arrays remain arrays; no spill).
20714            return Ok(smallvec::smallvec![Effect::WriteCell {
20715                vertex_id,
20716                value: computed_value,
20717            }]);
20718        }
20719
20720        match computed_value {
20721            LiteralValue::Array(rows) => self.plan_array_effects(vertex_id, rows),
20722            other => self.plan_scalar_effects(vertex_id, other),
20723        }
20724    }
20725
20726    /// Plan effects for a formula vertex that produced a scalar/error result.
20727    fn plan_scalar_effects(
20728        &mut self,
20729        vertex_id: VertexId,
20730        value: LiteralValue,
20731    ) -> Result<PlannedEffects, ExcelError> {
20732        // Range admission substitutes the same cap error before allocating rows.
20733        // Preserve the owned-array rejection's kind, release and clear effects.
20734        if let LiteralValue::Error(error) = &value
20735            && error.kind == ExcelErrorKind::Spill
20736            && error.message() == Some("SpillTooLarge")
20737            && let formualizer_common::ExcelErrorExtra::Spill {
20738                expected_rows,
20739                expected_cols,
20740            } = error.extra()
20741            && u64::from(*expected_rows).saturating_mul(u64::from(*expected_cols))
20742                > u64::from(self.config.spill.max_spill_cells)
20743        {
20744            self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20745            return self.plan_spill_error_effects(
20746                vertex_id,
20747                "SpillTooLarge",
20748                *expected_rows,
20749                *expected_cols,
20750            );
20751        }
20752        if !matches!(&value, LiteralValue::Error(e) if e.kind == ExcelErrorKind::Spill) {
20753            self.blocked_pending_spills
20754                .retain(|entry| entry.0 != vertex_id);
20755            self.spill_blocker_waiters.forget(vertex_id);
20756        }
20757        let has_spill = self
20758            .graph
20759            .spill_cells_for_anchor(vertex_id)
20760            .is_some_and(|c| !c.is_empty());
20761
20762        let mut effects = PlannedEffects::new();
20763        if has_spill {
20764            effects.push(Effect::SpillClear {
20765                anchor_vertex: vertex_id,
20766            });
20767        }
20768        effects.push(Effect::WriteCell { vertex_id, value });
20769        Ok(effects)
20770    }
20771
20772    /// Plan effects for a formula vertex that produced an array result.
20773    fn plan_array_effects(
20774        &mut self,
20775        vertex_id: VertexId,
20776        rows: Vec<Vec<LiteralValue>>,
20777    ) -> Result<PlannedEffects, ExcelError> {
20778        // Lightweight mutation needed for correct spill-blocking checks.
20779        self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20780
20781        let anchor = self
20782            .graph
20783            .get_cell_ref(vertex_id)
20784            .expect("cell ref for vertex");
20785        let sheet_id = anchor.sheet_id;
20786        let h = rows.len() as u32;
20787        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
20788
20789        // Hard cap to avoid vertex explosion from huge dynamic arrays.
20790        let spill_cells = (h as u64).saturating_mul(w as u64);
20791        if spill_cells > self.config.spill.max_spill_cells as u64 {
20792            return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
20793        }
20794
20795        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity).
20796        const PACKED_MAX_ROW: u32 = 1_048_575;
20797        const PACKED_MAX_COL: u32 = 16_383;
20798        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
20799        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
20800        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
20801            return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
20802        }
20803
20804        if h != 0 && w != 0 && self.graph.table_intersects_spill(anchor, end_row, end_col) {
20805            return self.plan_spill_error_effects(vertex_id, "Spill blocked by table", h, w);
20806        }
20807        let mut targets = Vec::new();
20808        for r in 0..h {
20809            for c in 0..w {
20810                targets.push(self.graph.make_cell_ref_internal(
20811                    sheet_id,
20812                    anchor.coord.row() + r,
20813                    anchor.coord.col() + c,
20814                ));
20815            }
20816        }
20817
20818        if h != 0 && w != 0 {
20819            let occupied = self.pending_spill_occupied(anchor, end_row, end_col);
20820            if occupied
20821                || self
20822                    .blocked_pending_spills
20823                    .iter()
20824                    .any(|entry| entry.0 == vertex_id)
20825            {
20826                self.spill_mgr.release_owner(vertex_id);
20827                self.remember_pending_spill(
20828                    vertex_id,
20829                    anchor,
20830                    Region::rect(
20831                        sheet_id,
20832                        anchor.coord.row(),
20833                        end_row,
20834                        anchor.coord.col(),
20835                        end_col,
20836                    ),
20837                )?;
20838            }
20839            if occupied {
20840                return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20841            }
20842        }
20843
20844        // Region lock via spill manager.
20845        match self.spill_mgr.reserve(
20846            vertex_id,
20847            anchor,
20848            SpillShape { rows: h, cols: w },
20849            SpillMeta {
20850                epoch: self.recalc_epoch,
20851                config: self.config.spill,
20852            },
20853        ) {
20854            Ok(()) => {
20855                // Validate spill region is available.
20856                let mut yield_effects = PlannedEffects::new();
20857                if let Err((e, blocker)) = self
20858                    .graph
20859                    .plan_spill_region_with_blocker(vertex_id, &targets)
20860                {
20861                    let yielding = if e.message() == Some("BlockedBySpill") {
20862                        self.graph.spills_yielding_to(vertex_id, &targets)
20863                    } else {
20864                        None
20865                    };
20866                    if let Some(yielding) = yielding {
20867                        // A colliding spill later in (sheet, column, row)
20868                        // order yields: it is cleared and replans to
20869                        // `#SPILL!` with its dependents.
20870                        for &other in &yielding {
20871                            let (rows, cols) = self
20872                                .graph
20873                                .spill_extent_for_anchor(other)
20874                                .map(|(first, last)| {
20875                                    (
20876                                        last.coord.row() - first.coord.row() + 1,
20877                                        last.coord.col() - first.coord.col() + 1,
20878                                    )
20879                                })
20880                                .unwrap_or((0, 0));
20881                            yield_effects.push(Effect::SpillClear {
20882                                anchor_vertex: other,
20883                            });
20884                            yield_effects.push(Effect::WriteCell {
20885                                vertex_id: other,
20886                                value: Self::spill_error_value("Spill blocked", rows, cols),
20887                            });
20888                        }
20889                        self.graph.mark_dirty_many(&yielding);
20890                    } else {
20891                        // A formula or spill blocker can go away without this
20892                        // anchor recomputing: remember the attempted region so
20893                        // an edit inside it wakes the anchor.
20894                        if matches!(e.message(), Some("BlockedByFormula" | "BlockedBySpill")) {
20895                            self.spill_blocker_waiters.wait(vertex_id, blocker);
20896                        }
20897                        return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20898                    }
20899                }
20900
20901                // Arrow-canonical mode: graph planning cannot see non-empty value blockers because
20902                // cell values are not cached in the dependency graph. Consult Arrow storage to
20903                // detect occupied cells in the target region.
20904                if !self.graph.value_cache_enabled() {
20905                    let sheet_name = self.graph.sheet_name(sheet_id);
20906                    if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
20907                        for cell in targets.iter() {
20908                            // Allow overwriting the anchor itself.
20909                            if *cell == anchor {
20910                                continue;
20911                            }
20912                            // Allow cells already owned by a spill (plan() validated spill ownership).
20913                            if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
20914                                continue;
20915                            }
20916                            // Skip formula blockers; plan() handled them (or allowed).
20917                            if let Some(vid) = self.graph.get_vertex_id_for_address(cell)
20918                                && vid != vertex_id
20919                            {
20920                                match self.graph.get_vertex_kind(vid) {
20921                                    VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20922                                        continue;
20923                                    }
20924                                    _ => {}
20925                                }
20926                            }
20927
20928                            let v = asheet.get_cell_value(
20929                                cell.coord.row() as usize,
20930                                cell.coord.col() as usize,
20931                            );
20932                            if !matches!(v, LiteralValue::Empty) {
20933                                return self.plan_spill_error_effects(
20934                                    vertex_id,
20935                                    "BlockedByValue",
20936                                    h,
20937                                    w,
20938                                );
20939                            }
20940                        }
20941                    }
20942                }
20943
20944                let top_left = rows
20945                    .first()
20946                    .and_then(|r| r.first())
20947                    .cloned()
20948                    .unwrap_or(LiteralValue::Empty);
20949
20950                let mut effects = yield_effects;
20951                // Clear previous spill if any.
20952                let has_prev = self
20953                    .graph
20954                    .spill_cells_for_anchor(vertex_id)
20955                    .is_some_and(|c| !c.is_empty());
20956                if has_prev {
20957                    effects.push(Effect::SpillClear {
20958                        anchor_vertex: vertex_id,
20959                    });
20960                }
20961                effects.push(Effect::SpillCommit {
20962                    anchor_vertex: vertex_id,
20963                    anchor_cell: anchor,
20964                    target_cells: targets,
20965                    values: rows,
20966                });
20967                effects.push(Effect::WriteCell {
20968                    vertex_id,
20969                    value: top_left,
20970                });
20971                Ok(effects)
20972            }
20973            Err(e) => {
20974                let msg = e.message().unwrap_or("Spill blocked");
20975                self.plan_spill_error_effects(vertex_id, msg, h, w)
20976            }
20977        }
20978    }
20979
20980    /// The `#SPILL!` value a blocked anchor holds.
20981    fn spill_error_value(message: &str, expected_rows: u32, expected_cols: u32) -> LiteralValue {
20982        LiteralValue::Error(
20983            ExcelError::new(ExcelErrorKind::Spill)
20984                .with_message(message)
20985                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20986                    expected_rows,
20987                    expected_cols,
20988                }),
20989        )
20990    }
20991
20992    /// Build the effect list for a spill that failed validation.
20993    fn plan_spill_error_effects(
20994        &mut self,
20995        vertex_id: VertexId,
20996        message: &str,
20997        expected_rows: u32,
20998        expected_cols: u32,
20999    ) -> Result<PlannedEffects, ExcelError> {
21000        self.spill_mgr.release_owner(vertex_id);
21001        let spill_val = Self::spill_error_value(message, expected_rows, expected_cols);
21002
21003        let effects = smallvec::smallvec![
21004            Effect::SpillClear {
21005                anchor_vertex: vertex_id,
21006            },
21007            Effect::WriteCell {
21008                vertex_id,
21009                value: spill_val,
21010            },
21011        ];
21012        Ok(effects)
21013    }
21014
21015    /// Apply a single effect, performing the actual graph mutations.
21016    pub(crate) fn apply_effect(
21017        &mut self,
21018        effect: &Effect,
21019        delta: Option<&mut DeltaCollector>,
21020        log: Option<&mut ChangeLog>,
21021    ) -> Result<(), ExcelError> {
21022        self.apply_effect_with_computed_writes(effect, delta, log, None)
21023    }
21024
21025    fn apply_effect_with_computed_writes(
21026        &mut self,
21027        effect: &Effect,
21028        delta: Option<&mut DeltaCollector>,
21029        log: Option<&mut ChangeLog>,
21030        computed_writes: Option<&mut ComputedWriteBuffer>,
21031    ) -> Result<(), ExcelError> {
21032        match effect {
21033            Effect::WriteCell { vertex_id, value } => {
21034                self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
21035            }
21036            Effect::SpillClear { anchor_vertex } => {
21037                self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
21038            }
21039            Effect::SpillCommit {
21040                anchor_vertex,
21041                anchor_cell: _,
21042                target_cells,
21043                values,
21044            } => {
21045                self.apply_spill_commit(
21046                    *anchor_vertex,
21047                    target_cells,
21048                    values.clone(),
21049                    delta,
21050                    log,
21051                    computed_writes,
21052                )?;
21053            }
21054        }
21055        Ok(())
21056    }
21057
21058    /// Apply a WriteCell effect.
21059    fn apply_write_cell(
21060        &mut self,
21061        vertex_id: VertexId,
21062        value: &LiteralValue,
21063        delta: Option<&mut DeltaCollector>,
21064        mut computed_writes: Option<&mut ComputedWriteBuffer>,
21065    ) -> Result<(), ExcelError> {
21066        if let Some(d) = delta
21067            && d.mode != DeltaMode::Off
21068        {
21069            if let Some(buffer) = computed_writes.as_deref_mut() {
21070                self.flush_computed_write_buffer(buffer)?;
21071            }
21072            if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
21073                let sheet_name = self.graph.sheet_name(cell.sheet_id);
21074                let old = self
21075                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21076                    .unwrap_or(LiteralValue::Empty);
21077                if old != *value {
21078                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
21079                }
21080            }
21081        }
21082        self.graph.update_vertex_value_ref(vertex_id, value);
21083        self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
21084        Ok(())
21085    }
21086
21087    /// Apply a SpillClear effect.
21088    fn apply_spill_clear(
21089        &mut self,
21090        anchor_vertex: VertexId,
21091        delta: Option<&mut DeltaCollector>,
21092        log: Option<&mut ChangeLog>,
21093        computed_writes: Option<&mut ComputedWriteBuffer>,
21094    ) -> Result<(), ExcelError> {
21095        if let Some(buffer) = computed_writes {
21096            self.flush_computed_write_buffer(buffer)?;
21097        }
21098
21099        let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
21100        if spill_cells.is_empty() {
21101            return Ok(());
21102        }
21103
21104        // Snapshot for ChangeLog before clearing.
21105        let snapshot = if log.is_some() {
21106            self.snapshot_spill_for_anchor(anchor_vertex)
21107        } else {
21108            None
21109        };
21110
21111        // Record delta for cleared cells.
21112        if let Some(d) = delta
21113            && d.mode != DeltaMode::Off
21114        {
21115            let empty = LiteralValue::Empty;
21116            for cell in spill_cells.iter() {
21117                let sheet_name = self.graph.sheet_name(cell.sheet_id);
21118                let old = self
21119                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21120                    .unwrap_or(LiteralValue::Empty);
21121                if old != empty {
21122                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
21123                }
21124            }
21125        }
21126
21127        self.graph.clear_spill_region(anchor_vertex);
21128        if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
21129            self.record_structural_change(scope);
21130        }
21131
21132        // Mirror Empty to Arrow overlay for cleared cells.
21133        if self.config.arrow_storage_enabled
21134            && self.config.delta_overlay_enabled
21135            && self.config.write_formula_overlay_enabled
21136        {
21137            let empty = LiteralValue::Empty;
21138            for cell in spill_cells.iter() {
21139                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
21140                self.mirror_value_to_computed_overlay(
21141                    &sheet_name,
21142                    cell.coord.row() + 1,
21143                    cell.coord.col() + 1,
21144                    &empty,
21145                );
21146            }
21147        }
21148
21149        // ChangeLog.
21150        if let Some(log) = log
21151            && let Some(old) = snapshot
21152        {
21153            log.record(ChangeEvent::SpillCleared {
21154                anchor: anchor_vertex,
21155                old,
21156            });
21157        }
21158        Ok(())
21159    }
21160
21161    /// Apply a SpillCommit effect.
21162    fn apply_spill_commit(
21163        &mut self,
21164        anchor_vertex: VertexId,
21165        target_cells: &[CellRef],
21166        values: Vec<Vec<LiteralValue>>,
21167        delta: Option<&mut DeltaCollector>,
21168        log: Option<&mut ChangeLog>,
21169        computed_writes: Option<&mut ComputedWriteBuffer>,
21170    ) -> Result<(), ExcelError> {
21171        self.guard_pending_spill_commit(anchor_vertex, target_cells)?;
21172        if let Some(buffer) = computed_writes {
21173            self.flush_computed_write_buffer(buffer)?;
21174        }
21175
21176        // Snapshot for ChangeLog before commit.
21177        let old_snapshot = if log.is_some() {
21178            self.snapshot_spill_for_anchor(anchor_vertex)
21179        } else {
21180            None
21181        };
21182
21183        // Delegate to existing commit_spill_and_mirror for delta + overlay logic.
21184        self.commit_spill_and_mirror(anchor_vertex, target_cells, values.clone(), delta)?;
21185
21186        // ChangeLog.
21187        if let Some(log) = log {
21188            log.record(ChangeEvent::SpillCommitted {
21189                anchor: anchor_vertex,
21190                old: old_snapshot,
21191                new: SpillSnapshot {
21192                    target_cells: target_cells.to_vec(),
21193                    values,
21194                },
21195            });
21196        }
21197        Ok(())
21198    }
21199
21200    /// Snapshot a spill region for ChangeLog recording.
21201    ///
21202    /// Extracted from `VertexEditor::snapshot_spill_for_anchor` to be usable
21203    /// without creating a `VertexEditor`.
21204    fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
21205        let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
21206        if cells.is_empty() {
21207            return None;
21208        }
21209
21210        let max = self.config.spill.max_spill_cells as usize;
21211        let mut cells = cells;
21212        if cells.len() > max {
21213            cells.truncate(max);
21214        }
21215
21216        let first = *cells.first().expect("non-empty spill cells");
21217        let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
21218        let row0 = first.coord.row();
21219        let col0 = first.coord.col();
21220
21221        let mut max_row = row0;
21222        let mut max_col = col0;
21223        let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
21224        for cell in &cells {
21225            max_row = max_row.max(cell.coord.row());
21226            max_col = max_col.max(cell.coord.col());
21227            let v = self
21228                .read_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21229                .unwrap_or(LiteralValue::Empty);
21230            by_coord.insert((cell.coord.row(), cell.coord.col()), v);
21231        }
21232
21233        let rows = (max_row - row0 + 1) as usize;
21234        let cols = (max_col - col0 + 1) as usize;
21235        let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
21236        for r in 0..rows {
21237            let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
21238            for c in 0..cols {
21239                row.push(
21240                    by_coord
21241                        .get(&(row0 + r as u32, col0 + c as u32))
21242                        .cloned()
21243                        .unwrap_or(LiteralValue::Empty),
21244                );
21245            }
21246            values.push(row);
21247        }
21248
21249        Some(SpillSnapshot {
21250            target_cells: cells,
21251            values,
21252        })
21253    }
21254
21255    fn flush_before_range_dependent_vertex(
21256        &mut self,
21257        vertex_id: VertexId,
21258        computed_writes: &mut ComputedWriteBuffer,
21259    ) -> Result<(), ExcelError> {
21260        if self.graph.reads_compressed_range(vertex_id) {
21261            self.flush_computed_write_buffer(computed_writes)?;
21262        }
21263        Ok(())
21264    }
21265
21266    fn plan_vertex_effects_with_computed_flush(
21267        &mut self,
21268        vertex_id: VertexId,
21269        computed_value: LiteralValue,
21270        computed_writes: &mut ComputedWriteBuffer,
21271    ) -> Result<PlannedEffects, ExcelError> {
21272        if matches!(&computed_value, LiteralValue::Array(_)) {
21273            self.flush_computed_write_buffer(computed_writes)?;
21274        }
21275        self.plan_vertex_effects(vertex_id, computed_value)
21276    }
21277
21278    // ── Layer evaluation via effects pipeline ──────────────────────────────
21279
21280    fn evaluate_small_layer_direct_effects(
21281        &mut self,
21282        layer: &super::scheduler::Layer,
21283        delta: Option<&mut DeltaCollector>,
21284        log: Option<&mut ChangeLog>,
21285        cancel_flag: Option<&AtomicBool>,
21286        cancel_check_every: usize,
21287        cancel_message: &'static str,
21288    ) -> Result<usize, ExcelError> {
21289        let cancel = cancel_flag.map(|flag| (flag, cancel_check_every, cancel_message));
21290        self.evaluate_layer_units(layer, delta, log, cancel, false)
21291    }
21292
21293    /// Sequential layer walk over its units (single cells and family runs):
21294    /// each unit evaluates, then its vertices' effects apply in order. With
21295    /// `buffered`, computed writes coalesce in a layer buffer (flushed before
21296    /// a unit that reads a compressed range, before array results, and at
21297    /// the end); otherwise they apply directly. `cancel` = (flag, check every
21298    /// N vertices, message).
21299    fn evaluate_layer_units(
21300        &mut self,
21301        layer: &super::scheduler::Layer,
21302        delta: Option<&mut DeltaCollector>,
21303        log: Option<&mut ChangeLog>,
21304        cancel: Option<(&AtomicBool, usize, &'static str)>,
21305        buffered: bool,
21306    ) -> Result<usize, ExcelError> {
21307        self.evaluate_layer_units_until(layer, delta, log, cancel, buffered, None)
21308    }
21309
21310    /// [`Self::evaluate_layer_units`] that stops before the next unit once
21311    /// `stop_at` has passed; returns the vertices evaluated (a prefix of
21312    /// the layer, all committed).
21313    fn evaluate_layer_units_until(
21314        &mut self,
21315        layer: &super::scheduler::Layer,
21316        mut delta: Option<&mut DeltaCollector>,
21317        mut log: Option<&mut ChangeLog>,
21318        cancel: Option<(&AtomicBool, usize, &'static str)>,
21319        buffered: bool,
21320        stop_at: Option<crate::instant::FzInstant>,
21321    ) -> Result<usize, ExcelError> {
21322        // A chain unit: its run through the chain lift, or else cell by
21323        // cell in row order, each written before the next reads it.
21324        let mut chain_values = None;
21325        if layer.sequential && !layer.runs.is_empty() {
21326            chain_values = match layer.runs.as_slice() {
21327                [run] if run.start == 0 && run.len as usize == layer.vertices.len() => {
21328                    self.try_chain_lift(*run, &layer.vertices)
21329                }
21330                _ => None,
21331            };
21332            if chain_values.is_none() {
21333                let cells = super::scheduler::Layer {
21334                    vertices: layer.vertices.clone(),
21335                    runs: Vec::new(),
21336                    sequential: true,
21337                };
21338                return self.evaluate_layer_units_until(&cells, delta, log, cancel, false, stop_at);
21339            }
21340        }
21341        let chained = chain_values.is_some();
21342        // The chain lift computed every member: one block write, as a run.
21343        let buffered = buffered || chained;
21344        // A dynamic reader's targets are not always ordered before it (its
21345        // pre-probe or observed reads can miss them, e.g. after a structural
21346        // edit). In a buffered layer a member's dirty flag is cleared at its
21347        // commit but its value written at the flush: the members committed
21348        // since the last flush count as dirty for the reader's freshness
21349        // check, which then re-plans it after them.
21350        let track_unflushed = buffered
21351            && self.freshness_armed()
21352            && layer.vertices.iter().any(|&v| self.graph.is_dynamic(v));
21353        let mut committed_unit: &[VertexId] = &[];
21354        let mut computed_writes = ComputedWriteBuffer::default();
21355        let mut next_check = 0usize;
21356        let mut done = 0usize;
21357        for unit in layer_units(layer) {
21358            if done > 0
21359                && let Some(stop_at) = stop_at
21360                && crate::instant::FzInstant::now() >= stop_at
21361            {
21362                self.flush_computed_write_buffer(&mut computed_writes)?;
21363                if track_unflushed {
21364                    self.freshness_flushed();
21365                }
21366                return Ok(done);
21367            }
21368            if let Some((flag, every, message)) = cancel
21369                && every > 0
21370                && done >= next_check
21371            {
21372                next_check = (done / every + 1) * every;
21373                if flag.load(Ordering::Relaxed) {
21374                    if buffered {
21375                        self.flush_computed_write_buffer(&mut computed_writes)?;
21376                    }
21377                    return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21378                        .with_message(message.to_string()));
21379                }
21380            }
21381            if buffered && self.unit_reads_compressed_range(layer, unit) {
21382                self.flush_computed_write_buffer(&mut computed_writes)?;
21383            }
21384            // The previous unit's members are committed (dirty flags
21385            // cleared); while their values wait in the buffer, a dynamic
21386            // reader's read of them is stale (`freshness_dirty_reads`).
21387            if track_unflushed {
21388                if computed_writes.is_empty() {
21389                    self.freshness_flushed();
21390                } else {
21391                    self.freshness_note_unflushed(committed_unit);
21392                }
21393                committed_unit = unit_members(layer, unit);
21394            }
21395            let values = match (chain_values.take(), unit) {
21396                (Some(chain), LayerUnit::Run(run)) => {
21397                    if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21398                        self.flush_computed_write_buffer(&mut computed_writes)?;
21399                        return Err(e);
21400                    }
21401                    let members =
21402                        &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21403                    let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21404                    match self.commit_run_numbers(
21405                        run,
21406                        members,
21407                        &chain,
21408                        delta_active,
21409                        Some(&mut computed_writes),
21410                    ) {
21411                        Ok(true) => {
21412                            done += chain.len();
21413                            continue;
21414                        }
21415                        Ok(false) => {}
21416                        Err(e) => {
21417                            self.flush_computed_write_buffer(&mut computed_writes)?;
21418                            return Err(e);
21419                        }
21420                    }
21421                    members
21422                        .iter()
21423                        .copied()
21424                        .zip(chain.into_iter().map(LiteralValue::Number))
21425                        .collect()
21426                }
21427                (_, unit) => self.evaluate_unit_immutable(layer, unit),
21428            };
21429            // Post-work boundary: the unit ran while (or after) the request
21430            // was cancelled; it is not committed and stays dirty.
21431            if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21432                self.flush_computed_write_buffer(&mut computed_writes)?;
21433                return Err(e);
21434            }
21435            done += values.len();
21436            if let LayerUnit::Run(run) = unit {
21437                let members = &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21438                let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21439                let committed = self.commit_run_scalars(
21440                    run,
21441                    members,
21442                    &values,
21443                    delta_active,
21444                    buffered.then_some(&mut computed_writes),
21445                );
21446                match committed {
21447                    Ok(true) => continue,
21448                    Ok(false) => {}
21449                    Err(e) => {
21450                        self.flush_computed_write_buffer(&mut computed_writes)?;
21451                        return Err(e);
21452                    }
21453                }
21454            }
21455            // A run unit's members were all evaluated before any commits
21456            // (FR5, FORM-192); a single cell is committed as it runs.
21457            let guarded = self.freshness_begin_batch_commit(&values);
21458            for (vertex_id, value) in values {
21459                let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21460                let effects = if buffered {
21461                    self.plan_vertex_effects_with_computed_flush(
21462                        vertex_id,
21463                        value,
21464                        &mut computed_writes,
21465                    )
21466                } else {
21467                    self.plan_vertex_effects(vertex_id, value)
21468                };
21469                let effects = match effects {
21470                    Ok(effects) => effects,
21471                    Err(e) => {
21472                        self.flush_computed_write_buffer(&mut computed_writes)?;
21473                        return Err(e);
21474                    }
21475                };
21476                for effect in &effects {
21477                    if let Err(e) = self.apply_effect_with_computed_writes(
21478                        effect,
21479                        delta.as_deref_mut(),
21480                        log.as_deref_mut(),
21481                        buffered.then_some(&mut computed_writes),
21482                    ) {
21483                        self.flush_computed_write_buffer(&mut computed_writes)?;
21484                        return Err(e);
21485                    }
21486                }
21487                self.freshness_keep_redirtied(redirtied, vertex_id);
21488            }
21489        }
21490        self.flush_computed_write_buffer(&mut computed_writes)?;
21491        if track_unflushed {
21492            self.freshness_flushed();
21493        }
21494        // Debug builds: every chain member equals the per-cell path, now
21495        // that the members above it are written.
21496        #[cfg(debug_assertions)]
21497        if chained {
21498            for &v in &layer.vertices {
21499                let cell = self.graph.get_cell_ref(v);
21500                let (sheet, row, col) = cell
21501                    .map(|c| {
21502                        (
21503                            self.graph.sheet_name(c.sheet_id).to_string(),
21504                            c.coord.row() + 1,
21505                            c.coord.col() + 1,
21506                        )
21507                    })
21508                    .expect("chain member cell");
21509                let written = self.read_cell_value(&sheet, row, col);
21510                let oracle = self
21511                    .evaluate_vertex_immutable(v)
21512                    .unwrap_or_else(LiteralValue::Error);
21513                assert!(
21514                    written
21515                        .as_ref()
21516                        .is_some_and(|w| same_value_bits(w, &oracle)),
21517                    "chain member {sheet}!R{row}C{col}: {written:?} vs per-cell {oracle:?}"
21518                );
21519            }
21520        }
21521        #[cfg(not(debug_assertions))]
21522        let _ = chained;
21523        Ok(layer.vertices.len())
21524    }
21525
21526    /// Evaluate a layer sequentially using the effects pipeline.
21527    fn evaluate_layer_sequential_effects(
21528        &mut self,
21529        layer: &super::scheduler::Layer,
21530    ) -> Result<usize, ExcelError> {
21531        let buffered = buffer_layer_writes(layer);
21532        self.evaluate_layer_units(layer, None, None, None, buffered)
21533    }
21534
21535    /// Evaluate a layer sequentially with delta collection via effects pipeline.
21536    fn evaluate_layer_sequential_with_delta_effects(
21537        &mut self,
21538        layer: &super::scheduler::Layer,
21539        delta: &mut DeltaCollector,
21540    ) -> Result<usize, ExcelError> {
21541        let buffered = buffer_layer_writes(layer);
21542        self.evaluate_layer_units(layer, Some(delta), None, None, buffered)
21543    }
21544
21545    /// Evaluate a layer sequentially with cancellation support via effects pipeline.
21546    fn evaluate_layer_sequential_cancellable_effects(
21547        &mut self,
21548        layer: &super::scheduler::Layer,
21549        cancel_flag: &AtomicBool,
21550    ) -> Result<usize, ExcelError> {
21551        let buffered = buffer_layer_writes(layer);
21552        let cancel = (cancel_flag, 256, "Evaluation cancelled within layer");
21553        self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21554    }
21555
21556    /// Evaluate a layer sequentially with more frequent cancellation for demand-driven eval.
21557    fn evaluate_layer_sequential_cancellable_demand_driven_effects(
21558        &mut self,
21559        layer: &super::scheduler::Layer,
21560        cancel_flag: &AtomicBool,
21561    ) -> Result<usize, ExcelError> {
21562        let buffered = buffer_layer_writes(layer);
21563        let cancel = (
21564            cancel_flag,
21565            128,
21566            "Demand-driven evaluation cancelled within layer",
21567        );
21568        self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21569    }
21570
21571    /// Evaluate a layer in parallel, applying via effects pipeline.
21572    fn evaluate_layer_parallel_effects(
21573        &mut self,
21574        layer: &super::scheduler::Layer,
21575        min_chunk: u32,
21576    ) -> Result<usize, ExcelError> {
21577        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21578
21579        let phases = self.parallel_phases(layer);
21580        let mut applied = 0usize;
21581
21582        for (units, group) in &phases {
21583            let group = &group[..];
21584            if group.is_empty() {
21585                continue;
21586            }
21587            let mut computed_writes = ComputedWriteBuffer::default();
21588
21589            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21590                thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, min_chunk));
21591
21592            // Post-work boundary: a group evaluated across a live
21593            // cancellation is not committed (it stays dirty).
21594            let results = results.and_then(|results| {
21595                self.live_cancellation_after_work(GROUP_CANCELLED)
21596                    .map(|()| results)
21597            });
21598            // FR3: a parallel group is one commit unit; one stale reader
21599            // drops the whole group (it stays dirty and replans).
21600            self.freshness_gate_group(group);
21601            match results {
21602                Ok(vertex_results) => {
21603                    let (vertex_results, committed) = self.commit_parallel_runs(
21604                        layer,
21605                        units,
21606                        vertex_results,
21607                        false,
21608                        &mut computed_writes,
21609                    )?;
21610                    applied = applied.saturating_add(committed);
21611                    // Arrays first, then scalars — establishes spill regions before
21612                    // scalar results that might land inside a spilled region.
21613                    // FR5 (FORM-192): members evaluated together must not
21614                    // clear a re-dirty from a spill committed before them.
21615                    let guarded = self.freshness_begin_batch_commit(&vertex_results);
21616                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21617                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21618                    for (vertex_id, result) in vertex_results {
21619                        if matches!(result, LiteralValue::Array(_)) {
21620                            arrays.push((vertex_id, result));
21621                        } else {
21622                            others.push((vertex_id, result));
21623                        }
21624                    }
21625                    for (vertex_id, result) in arrays {
21626                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21627                        let effects = match self.plan_vertex_effects_with_computed_flush(
21628                            vertex_id,
21629                            result,
21630                            &mut computed_writes,
21631                        ) {
21632                            Ok(effects) => effects,
21633                            Err(e) => {
21634                                self.flush_computed_write_buffer(&mut computed_writes)?;
21635                                return Err(e);
21636                            }
21637                        };
21638                        for effect in &effects {
21639                            if let Err(e) = self.apply_effect_with_computed_writes(
21640                                effect,
21641                                None,
21642                                None,
21643                                Some(&mut computed_writes),
21644                            ) {
21645                                self.flush_computed_write_buffer(&mut computed_writes)?;
21646                                return Err(e);
21647                            }
21648                        }
21649                        self.freshness_keep_redirtied(redirtied, vertex_id);
21650                        applied = applied.saturating_add(1);
21651                    }
21652                    // Make all array spill/top-left writes visible before scalar effects in this group.
21653                    self.flush_computed_write_buffer(&mut computed_writes)?;
21654                    for (vertex_id, result) in others {
21655                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21656                        let effects = match self.plan_vertex_effects_with_computed_flush(
21657                            vertex_id,
21658                            result,
21659                            &mut computed_writes,
21660                        ) {
21661                            Ok(effects) => effects,
21662                            Err(e) => {
21663                                self.flush_computed_write_buffer(&mut computed_writes)?;
21664                                return Err(e);
21665                            }
21666                        };
21667                        for effect in &effects {
21668                            if let Err(e) = self.apply_effect_with_computed_writes(
21669                                effect,
21670                                None,
21671                                None,
21672                                Some(&mut computed_writes),
21673                            ) {
21674                                self.flush_computed_write_buffer(&mut computed_writes)?;
21675                                return Err(e);
21676                            }
21677                        }
21678                        self.freshness_keep_redirtied(redirtied, vertex_id);
21679                        applied = applied.saturating_add(1);
21680                    }
21681                    // Flush at the group boundary; phase1 must be visible before phase2.
21682                    self.flush_computed_write_buffer(&mut computed_writes)?;
21683                }
21684                Err(e) => {
21685                    self.flush_computed_write_buffer(&mut computed_writes)?;
21686                    return Err(e);
21687                }
21688            }
21689        }
21690
21691        Ok(applied)
21692    }
21693
21694    /// Evaluate a layer in parallel with delta collection via effects pipeline.
21695    fn evaluate_layer_parallel_with_delta_effects(
21696        &mut self,
21697        layer: &super::scheduler::Layer,
21698        delta: &mut DeltaCollector,
21699    ) -> Result<usize, ExcelError> {
21700        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21701
21702        let phases = self.parallel_phases(layer);
21703        let mut applied = 0usize;
21704
21705        for (units, group) in &phases {
21706            let group = &group[..];
21707            if group.is_empty() {
21708                continue;
21709            }
21710            let mut computed_writes = ComputedWriteBuffer::default();
21711            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21712                thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, 8));
21713
21714            // Post-work boundary: a group evaluated across a live
21715            // cancellation is not committed (it stays dirty).
21716            let results = results.and_then(|results| {
21717                self.live_cancellation_after_work(GROUP_CANCELLED)
21718                    .map(|()| results)
21719            });
21720            // FR3: a parallel group is one commit unit; one stale reader
21721            // drops the whole group (it stays dirty and replans).
21722            self.freshness_gate_group(group);
21723            match results {
21724                Ok(vertex_results) => {
21725                    let (vertex_results, committed) = self.commit_parallel_runs(
21726                        layer,
21727                        units,
21728                        vertex_results,
21729                        delta.mode != DeltaMode::Off,
21730                        &mut computed_writes,
21731                    )?;
21732                    applied = applied.saturating_add(committed);
21733                    // FR5 (FORM-192): members evaluated together must not
21734                    // clear a re-dirty from a spill committed before them.
21735                    let guarded = self.freshness_begin_batch_commit(&vertex_results);
21736                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21737                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21738                    for (vertex_id, result) in vertex_results {
21739                        if matches!(result, LiteralValue::Array(_)) {
21740                            arrays.push((vertex_id, result));
21741                        } else {
21742                            others.push((vertex_id, result));
21743                        }
21744                    }
21745                    for (vertex_id, result) in arrays {
21746                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21747                        let effects = match self.plan_vertex_effects_with_computed_flush(
21748                            vertex_id,
21749                            result,
21750                            &mut computed_writes,
21751                        ) {
21752                            Ok(effects) => effects,
21753                            Err(e) => {
21754                                self.flush_computed_write_buffer(&mut computed_writes)?;
21755                                return Err(e);
21756                            }
21757                        };
21758                        for effect in &effects {
21759                            if let Err(e) = self.apply_effect_with_computed_writes(
21760                                effect,
21761                                Some(delta),
21762                                None,
21763                                Some(&mut computed_writes),
21764                            ) {
21765                                self.flush_computed_write_buffer(&mut computed_writes)?;
21766                                return Err(e);
21767                            }
21768                        }
21769                        self.freshness_keep_redirtied(redirtied, vertex_id);
21770                        applied = applied.saturating_add(1);
21771                    }
21772                    self.flush_computed_write_buffer(&mut computed_writes)?;
21773                    for (vertex_id, result) in others {
21774                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21775                        let effects = match self.plan_vertex_effects_with_computed_flush(
21776                            vertex_id,
21777                            result,
21778                            &mut computed_writes,
21779                        ) {
21780                            Ok(effects) => effects,
21781                            Err(e) => {
21782                                self.flush_computed_write_buffer(&mut computed_writes)?;
21783                                return Err(e);
21784                            }
21785                        };
21786                        for effect in &effects {
21787                            if let Err(e) = self.apply_effect_with_computed_writes(
21788                                effect,
21789                                Some(delta),
21790                                None,
21791                                Some(&mut computed_writes),
21792                            ) {
21793                                self.flush_computed_write_buffer(&mut computed_writes)?;
21794                                return Err(e);
21795                            }
21796                        }
21797                        self.freshness_keep_redirtied(redirtied, vertex_id);
21798                        applied = applied.saturating_add(1);
21799                    }
21800                    self.flush_computed_write_buffer(&mut computed_writes)?;
21801                }
21802                Err(e) => {
21803                    self.flush_computed_write_buffer(&mut computed_writes)?;
21804                    return Err(e);
21805                }
21806            }
21807        }
21808
21809        Ok(applied)
21810    }
21811
21812    /// Evaluate a layer in parallel with cancellation support via effects pipeline.
21813    fn evaluate_layer_parallel_cancellable_effects(
21814        &mut self,
21815        layer: &super::scheduler::Layer,
21816        cancel_flag: &AtomicBool,
21817    ) -> Result<usize, ExcelError> {
21818        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21819
21820        if cancel_flag.load(Ordering::Relaxed) {
21821            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21822                .with_message("Parallel evaluation cancelled before starting".to_string()));
21823        }
21824
21825        let phases = self.parallel_phases(layer);
21826        let mut applied = 0usize;
21827
21828        for (units, group) in &phases {
21829            let group = &group[..];
21830            if group.is_empty() {
21831                continue;
21832            }
21833            let mut computed_writes = ComputedWriteBuffer::default();
21834
21835            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> = thread_pool
21836                .install(|| self.evaluate_units_parallel(layer, units, Some(cancel_flag), 8));
21837
21838            // Post-work boundary: a group evaluated across a live
21839            // cancellation is not committed (it stays dirty).
21840            let results = results.and_then(|results| {
21841                self.live_cancellation_after_work(GROUP_CANCELLED)
21842                    .map(|()| results)
21843            });
21844            // FR3: a parallel group is one commit unit; one stale reader
21845            // drops the whole group (it stays dirty and replans).
21846            self.freshness_gate_group(group);
21847            match results {
21848                Ok(vertex_results) => {
21849                    let (vertex_results, committed) = self.commit_parallel_runs(
21850                        layer,
21851                        units,
21852                        vertex_results,
21853                        false,
21854                        &mut computed_writes,
21855                    )?;
21856                    applied = applied.saturating_add(committed);
21857                    // FR5 (FORM-192): members evaluated together must not
21858                    // clear a re-dirty from a spill committed before them.
21859                    let guarded = self.freshness_begin_batch_commit(&vertex_results);
21860                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21861                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21862                    for (vertex_id, result) in vertex_results {
21863                        if matches!(result, LiteralValue::Array(_)) {
21864                            arrays.push((vertex_id, result));
21865                        } else {
21866                            others.push((vertex_id, result));
21867                        }
21868                    }
21869                    for (vertex_id, result) in arrays {
21870                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21871                        let effects = match self.plan_vertex_effects_with_computed_flush(
21872                            vertex_id,
21873                            result,
21874                            &mut computed_writes,
21875                        ) {
21876                            Ok(effects) => effects,
21877                            Err(e) => {
21878                                self.flush_computed_write_buffer(&mut computed_writes)?;
21879                                return Err(e);
21880                            }
21881                        };
21882                        for effect in &effects {
21883                            if let Err(e) = self.apply_effect_with_computed_writes(
21884                                effect,
21885                                None,
21886                                None,
21887                                Some(&mut computed_writes),
21888                            ) {
21889                                self.flush_computed_write_buffer(&mut computed_writes)?;
21890                                return Err(e);
21891                            }
21892                        }
21893                        self.freshness_keep_redirtied(redirtied, vertex_id);
21894                        applied = applied.saturating_add(1);
21895                    }
21896                    self.flush_computed_write_buffer(&mut computed_writes)?;
21897                    for (vertex_id, result) in others {
21898                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21899                        let effects = match self.plan_vertex_effects_with_computed_flush(
21900                            vertex_id,
21901                            result,
21902                            &mut computed_writes,
21903                        ) {
21904                            Ok(effects) => effects,
21905                            Err(e) => {
21906                                self.flush_computed_write_buffer(&mut computed_writes)?;
21907                                return Err(e);
21908                            }
21909                        };
21910                        for effect in &effects {
21911                            if let Err(e) = self.apply_effect_with_computed_writes(
21912                                effect,
21913                                None,
21914                                None,
21915                                Some(&mut computed_writes),
21916                            ) {
21917                                self.flush_computed_write_buffer(&mut computed_writes)?;
21918                                return Err(e);
21919                            }
21920                        }
21921                        self.freshness_keep_redirtied(redirtied, vertex_id);
21922                        applied = applied.saturating_add(1);
21923                    }
21924                    self.flush_computed_write_buffer(&mut computed_writes)?;
21925                }
21926                Err(e) => {
21927                    self.flush_computed_write_buffer(&mut computed_writes)?;
21928                    return Err(e);
21929                }
21930            }
21931        }
21932
21933        Ok(applied)
21934    }
21935
21936    // ── Top-level evaluate_all_logged ───────────────────────────────────────
21937
21938    /// Evaluate all dirty/volatile vertices, recording effects into a ChangeLog.
21939    ///
21940    /// This is the same flow as `evaluate_all` but threads a ChangeLog through
21941    /// every effect application so that spill commits/clears are captured.
21942    pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
21943        self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
21944            engine.evaluate_all_logged_unobserved(log)
21945        })
21946    }
21947
21948    fn evaluate_all_logged_unobserved(
21949        &mut self,
21950        log: &mut ChangeLog,
21951    ) -> Result<EvalResult, ExcelError> {
21952        self.observe_function_semantic_epoch()?;
21953        let _source_cache = self.source_cache_session();
21954        self.validate_deterministic_mode()?;
21955        if self.config.defer_graph_building {
21956            self.build_graph_all()?;
21957        }
21958        self.require_unified_authority()?;
21959        self.begin_evaluation_request();
21960        self.reset_virtual_dep_telemetry_if_disabled();
21961        let start = crate::instant::FzInstant::now();
21962        let mut computed_vertices = 0;
21963        let mut cycle_errors = 0;
21964
21965        let mut replan_iterations = 0;
21966        const MAX_REPLAN: usize = 5;
21967        let mut telemetry = self
21968            .config
21969            .enable_virtual_dep_telemetry
21970            .then(|| self.start_virtual_dep_telemetry());
21971
21972        log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
21973
21974        let result = (|| -> Result<EvalResult, ExcelError> {
21975            loop {
21976                let to_evaluate = self.graph.get_evaluation_vertices();
21977                if to_evaluate.is_empty() {
21978                    if let Some(t) = telemetry.as_mut()
21979                        && t.bailout_reason.is_none()
21980                    {
21981                        t.bailout_reason = Some("no_work");
21982                    }
21983                    break;
21984                }
21985
21986                let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21987                if let Some(t) = telemetry.as_mut() {
21988                    Self::accumulate_schedule_meta(t, &meta);
21989                }
21990
21991                // Walk units in condensation order: stamp cycles at their
21992                // position, evaluate layers with ChangeLog recording.
21993                self.begin_pass(&schedule);
21994                for (unit_index, &unit) in schedule.units.iter().enumerate() {
21995                    match unit {
21996                        ScheduleUnit::Cycle(i) => {
21997                            // Journal integration (design doc §4 last row): the
21998                            // ChangeLog in this path only records SpillClear /
21999                            // SpillCommit events; WriteCell effects are never
22000                            // logged (see `apply_write_cell`). Runtime SCC tasks
22001                            // write values directly and never spill (§7.9 stamps
22002                            // would-be anchors), and their spill *teardown* is the
22003                            // same unlogged `stamp_cycle_error` the Static path
22004                            // already uses here — so direct commits coexist with
22005                            // the journal cleanly, with identical semantics to
22006                            // Static. Pinned by `scc_runtime_cycles` tests.
22007                            if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
22008                            {
22009                                cycle_errors += 1;
22010                            }
22011                        }
22012                        ScheduleUnit::Layer(i) => {
22013                            computed_vertices +=
22014                                self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
22015                        }
22016                    }
22017                    if self.stop_after_unit(&schedule, unit_index) {
22018                        break;
22019                    }
22020                }
22021
22022                let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
22023                if let Some(t) = telemetry.as_mut() {
22024                    t.changed_vdeps_total += changed_vertices.len();
22025                }
22026                self.resource_checkpoint(0)?;
22027                if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
22028                    if let Some(t) = telemetry.as_mut() {
22029                        t.bailout_reason = Some("converged");
22030                    }
22031                    break;
22032                }
22033                if replan_iterations >= MAX_REPLAN {
22034                    if let Some(mut t) = telemetry.take() {
22035                        t.bailout_reason = Some("max_replan");
22036                        t.replan_iterations = replan_iterations;
22037                        self.last_virtual_dep_telemetry = t;
22038                    }
22039                    return Err(
22040                        self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
22041                    );
22042                }
22043                replan_iterations += 1;
22044            }
22045
22046            if let Some(mut t) = telemetry {
22047                t.replan_iterations = replan_iterations;
22048                self.last_virtual_dep_telemetry = t;
22049            }
22050
22051            self.redirty_for_next_recalc();
22052            self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
22053
22054            Ok(EvalResult {
22055                computed_vertices,
22056                cycle_errors,
22057                elapsed: start.elapsed(),
22058            })
22059        })();
22060        log.end_compound();
22061        result
22062    }
22063
22064    /// Evaluate a single layer with ChangeLog recording.
22065    fn evaluate_layer_logged(
22066        &mut self,
22067        layer: &super::scheduler::Layer,
22068        log: &mut ChangeLog,
22069    ) -> Result<usize, ExcelError> {
22070        self.resource_checkpoint(layer.vertices.len() as u64)?;
22071        self.evaluate_layer_units(layer, None, Some(log), None, true)
22072    }
22073}