Skip to main content

formualizer_eval/engine/
eval.rs

1use crate::SheetId;
2
3mod criteria;
4mod criteria_mask_cache;
5#[cfg(test)]
6mod criteria_mask_cache_tests;
7mod exact;
8mod family;
9mod kernels;
10mod lift;
11mod memo;
12use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
13#[cfg(test)]
14use crate::engine::Scheduler;
15use crate::engine::arena::AstNodeId;
16use crate::engine::eval_delta::{
17    DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
18};
19use crate::engine::graph::editor::change_log::MutationCapture;
20use crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan;
21use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
22use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
23use crate::engine::live_graph::analyze_live_graph;
24use crate::engine::lookup_index_cache::{
25    BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
26    LookupIndexKey, estimate_bytes,
27};
28use crate::engine::named_range::{NameScope, NamedDefinition};
29use crate::engine::range_view::RangeView;
30use crate::engine::row_visibility::RowVisibilityState;
31use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
32use crate::engine::target_preparation::{
33    StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
34};
35use crate::engine::used_extent::{
36    ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
37};
38use crate::engine::virtual_deps::VirtualDepBuilder;
39use family::{LayerUnit, layer_units, unit_members};
40
41#[path = "freshness.rs"]
42mod freshness;
43use crate::engine::template::region::Region;
44use crate::engine::{
45    ChangeLogger, CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
46    EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceRequestStats,
47    FormulaDirtyLeaseOutcome, FormulaIngestBatch, FormulaIngestRecord, FormulaIngestReport,
48    FormulaParseDiagnostic, FormulaParsePolicy, FormulaPlaneMode, ResourceLedger,
49    RowVisibilitySource, ScheduleUnit, VertexId, VertexKind, VisibilityMaskMode,
50};
51use crate::function::FnCaps;
52use crate::interpreter::Interpreter;
53
54#[cfg(test)]
55mod fn_snapshot_tests;
56use crate::reference::{CellRef, Coord, RangeRef};
57use crate::traits::FunctionProvider;
58use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
59use formualizer_common::{
60    CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
61};
62use formualizer_parse::parser::ReferenceType;
63use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
64use rayon::ThreadPoolBuilder;
65use rustc_hash::{FxHashMap, FxHashSet};
66use std::collections::{BTreeMap, BTreeSet, VecDeque};
67use std::sync::Arc;
68use std::sync::atomic::{AtomicBool, Ordering};
69use std::time::Instant;
70
71type StagedFormulaEntry = (u32, u32, String);
72type StagedSheetParts = (
73    Vec<StagedFormulaEntry>,
74    Option<crate::engine::DeferredFormulaPackage>,
75);
76
77/// Per-sheet staged-formula store (NOTE(#126) follow-up).
78///
79/// Ingest consumers (`build_graph_all`/`build_graph_for_sheets`) walk staged
80/// entries in INSERTION order, so the order-preserving `Vec` stays the
81/// canonical storage; a `(row, col) → index` map removes the linear dup-scan
82/// that made `stage_formula_text`/`get_staged_formula_text` O(staged-on-sheet)
83/// per call (O(n²) for an n-formula deferred load on one sheet — ~570 ms for
84/// 50k stages, release, before the index). `stage`/`get` are O(1);
85/// `remove` keeps the old O(n) `Vec::remove` (rare path, order preserved).
86#[derive(Default)]
87pub(crate) struct StagedSheet {
88    entries: Vec<StagedFormulaEntry>,
89    index: FxHashMap<(u32, u32), usize>,
90    deferred_package: Option<crate::engine::DeferredFormulaPackage>,
91}
92
93impl StagedSheet {
94    fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
95        package
96            .invalidated
97            .extend(package.families.iter().map(|family| family.source_id));
98        package.invalidated.extend(
99            package
100                .partitioned_families
101                .iter()
102                .map(|family| family.source_id),
103        );
104    }
105
106    fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
107        let Some(package) = self.deferred_package.as_mut() else {
108            return;
109        };
110        if package.suppressed.contains(&(row, col)) {
111            // Previously consumed coordinates are already excluded from residual
112            // authority. Edits/deletion must not invalidate untouched fragments.
113            return;
114        }
115        if package.source_geometry_complete
116            && package.families.is_empty()
117            && package.partitioned_families.is_empty()
118        {
119            if let Some((row0, col0)) = row.checked_sub(1).zip(col.checked_sub(1))
120                && package
121                    .source_coordinates
122                    .binary_search(&crate::engine::SourceCoord {
123                        row: row0,
124                        col: col0,
125                    })
126                    .is_ok()
127            {
128                package.suppressed.insert((row, col));
129            }
130            return;
131        }
132        let family = package
133            .replay
134            .lock()
135            .ok()
136            .and_then(|mut replay| replay.formula_at(row, col).ok())
137            .flatten()
138            .and_then(|record| record.partition_owner.or(record.family));
139        if let Some(family) = family {
140            package.invalidated.insert(family);
141        } else {
142            // A poisoned lock, replay failure, malformed spool, or missing
143            // lookup result must never let a possibly edited family commit.
144            // Fail closed at package scope before suppressing the edited cell.
145            Self::invalidate_all_deferred_families(package);
146        }
147        package.suppressed.insert((row, col));
148    }
149
150    fn reconcile_attached_deferred_package(&mut self) {
151        let Some(package) = self.deferred_package.as_mut() else {
152            return;
153        };
154        if self.entries.is_empty() {
155            return;
156        }
157
158        let replayed = (|| {
159            let mut disposition = crate::engine::FormulaReplayDisposition::default();
160            for partition in &package.partitioned_families {
161                disposition.register_partition(partition, false)?;
162            }
163            package
164                .replay
165                .lock()
166                .map_err(|_| "deferred formula spool lock poisoned".to_string())?
167                .replay_partitioned(&disposition, &package.partitioned_families)
168        })();
169
170        let Ok(records) = replayed else {
171            Self::invalidate_all_deferred_families(package);
172            package
173                .suppressed
174                .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
175            return;
176        };
177        let mut owners = FxHashMap::default();
178        for record in &records {
179            owners
180                .entry((record.row, record.col))
181                .or_insert(record.partition_owner.or(record.family));
182        }
183        for (row, col, _) in &self.entries {
184            if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
185                package.invalidated.insert(family);
186            } else {
187                Self::invalidate_all_deferred_families(package);
188            }
189            package.suppressed.insert((*row, *col));
190        }
191        package.reconciliation_replay = Some(records);
192    }
193
194    fn stage(&mut self, row: u32, col: u32, text: String) {
195        self.invalidate_deferred_at(row, col);
196        match self.index.entry((row, col)) {
197            std::collections::hash_map::Entry::Occupied(slot) => {
198                self.entries[*slot.get()].2 = text;
199            }
200            std::collections::hash_map::Entry::Vacant(slot) => {
201                slot.insert(self.entries.len());
202                self.entries.push((row, col, text));
203            }
204        }
205    }
206
207    fn remove(&mut self, row: u32, col: u32) -> Option<String> {
208        let deferred_text = self.get(row, col);
209        self.invalidate_deferred_at(row, col);
210        let Some(idx) = self.index.remove(&(row, col)) else {
211            return deferred_text;
212        };
213        let (_, _, text) = self.entries.remove(idx);
214        // `Vec::remove` shifted everything after `idx` left by one.
215        for slot in self.index.values_mut() {
216            if *slot > idx {
217                *slot -= 1;
218            }
219        }
220        Some(text)
221    }
222
223    fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
224        self.index
225            .get(&(row, col))
226            .map(|&i| self.entries[i].2.as_str())
227    }
228
229    fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
230        let idx = self.index.remove(&(row, col))?;
231        let (_, _, text) = self.entries.remove(idx);
232        for slot in self.index.values_mut() {
233            if *slot > idx {
234                *slot -= 1;
235            }
236        }
237        Some(text)
238    }
239
240    fn get(&self, row: u32, col: u32) -> Option<String> {
241        if let Some(text) = self.get_ordinary(row, col) {
242            return Some(text.to_string());
243        }
244        let package = self.deferred_package.as_ref()?;
245        if package.suppressed.contains(&(row, col)) {
246            return None;
247        }
248        package
249            .replay
250            .lock()
251            .ok()?
252            .formula_at(row, col)
253            .ok()?
254            .map(|record| record.text)
255    }
256
257    fn len(&self) -> usize {
258        self.deferred_package
259            .as_ref()
260            .map_or(self.entries.len(), |package| {
261                (if package.source_geometry_complete
262                    && (package.coordinates_cover_families
263                        || (package.families.is_empty() && package.partitioned_families.is_empty()))
264                {
265                    package.source_coordinates.len()
266                } else {
267                    usize::try_from(package.report.source_formula_records_spooled)
268                        .unwrap_or(usize::MAX)
269                })
270                .saturating_sub(package.suppressed.len())
271                .saturating_add(self.entries.len())
272            })
273    }
274
275    fn is_empty(&self) -> bool {
276        self.entries.is_empty() && self.deferred_package.is_none()
277    }
278
279    fn into_parts(self) -> StagedSheetParts {
280        (self.entries, self.deferred_package)
281    }
282}
283
284type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
285
286#[derive(Clone, Debug, PartialEq, Eq, Hash)]
287struct PreparationRegion {
288    sheet: String,
289    sheet_id: SheetId,
290    start_row: u32,
291    start_col: u32,
292    end_row: u32,
293    end_col: u32,
294}
295
296// Target roots dedupe by identity; they keep their first-seen order.
297struct OrderedTargetProducers {
298    ordered: Vec<crate::engine::target_preparation::TargetProducer>,
299    seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
300}
301
302impl OrderedTargetProducers {
303    fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
304        let mut ordered = Vec::new();
305        ordered.try_reserve(capacity)?;
306        let mut seen = FxHashSet::default();
307        seen.try_reserve(capacity)?;
308        Ok(Self { ordered, seen })
309    }
310
311    fn from_ordered(
312        ordered: Vec<crate::engine::target_preparation::TargetProducer>,
313    ) -> Result<Self, std::collections::TryReserveError> {
314        let mut seen = FxHashSet::default();
315        seen.try_reserve(ordered.len())?;
316        seen.extend(ordered.iter().copied());
317        Ok(Self { ordered, seen })
318    }
319
320    fn push(
321        &mut self,
322        producer: crate::engine::target_preparation::TargetProducer,
323    ) -> Result<bool, std::collections::TryReserveError> {
324        #[cfg(test)]
325        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
326        if self.seen.contains(&producer) {
327            return Ok(false);
328        }
329        self.seen.try_reserve(1)?;
330        self.ordered.try_reserve(1)?;
331        self.seen.insert(producer);
332        self.ordered.push(producer);
333        Ok(true)
334    }
335
336    fn len(&self) -> usize {
337        self.ordered.len()
338    }
339
340    fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
341        self.ordered
342    }
343}
344
345#[cfg(test)]
346thread_local! {
347    static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
348}
349
350fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
351    crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
352        reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
353        limit: u64::MAX,
354        observed: observed as u64,
355        request_id,
356    })
357    .into_excel_error()
358}
359
360#[derive(Debug)]
361struct PreparedOrdinaryStagedFormula {
362    sheet: String,
363    sheet_id: SheetId,
364    lease: StagedFormulaLease,
365    ast_id: Option<AstNodeId>,
366    plan: Option<DependencyPlanRow>,
367}
368
369struct PreparedTargetSourcePackage {
370    sheet: String,
371    sheet_id: SheetId,
372    lease: StagedPackageLease,
373    selected_points: Option<BTreeSet<(u32, u32)>>,
374    complete_selections: BTreeSet<crate::engine::SourceFamilyId>,
375    deferred_shared: bool,
376    direct_domains: Vec<(
377        crate::engine::SourceFamilyId,
378        crate::engine::PlacementDomainTransport,
379    )>,
380    source_report: crate::engine::FormulaCompressedSourceReport,
381    replay_records: Vec<crate::engine::DeferredReplayFormula>,
382    spool_replays: u64,
383    disposition: crate::engine::FormulaReplayDisposition,
384    legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
385    direct_families: usize,
386    direct_cells: u64,
387    direct_fragments: u64,
388    direct_complete_families: u64,
389    direct_complete_cells: u64,
390    direct_partition_families: u64,
391    direct_partition_cells: u64,
392    anchor_parses: u64,
393    anchor_asts: u64,
394    anchor_analyses: u64,
395}
396
397impl PreparedTargetSourcePackage {
398    fn empty_selection(sheet: &str, sheet_id: SheetId, lease: StagedPackageLease) -> Self {
399        Self {
400            sheet: sheet.to_owned(),
401            sheet_id,
402            lease,
403            selected_points: Some(BTreeSet::new()),
404            complete_selections: Default::default(),
405            deferred_shared: false,
406            direct_domains: Vec::new(),
407            source_report: Default::default(),
408            replay_records: Vec::new(),
409            spool_replays: 0,
410            disposition: Default::default(),
411            legacy: Vec::new(),
412            direct_families: 0,
413            direct_cells: 0,
414            direct_fragments: 0,
415            direct_complete_families: 0,
416            direct_complete_cells: 0,
417            direct_partition_families: 0,
418            direct_partition_cells: 0,
419            anchor_parses: 0,
420            anchor_asts: 0,
421            anchor_analyses: 0,
422        }
423    }
424
425    fn direct_contains(&self, row: u32, col: u32) -> bool {
426        self.direct_domains.iter().any(|(_, domain)| {
427            let rect = domain.rect();
428            row > rect.start.row
429                && row <= rect.end.row + 1
430                && col > rect.start.col
431                && col <= rect.end.col + 1
432        })
433    }
434
435    fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
436        self.replay_records.iter().filter(|record| {
437            let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
438                return true;
439            };
440            let coord = crate::engine::SourceCoord { row, col };
441            let disposition = match record.family {
442                Some(family) => self.disposition.shared_disposition(family, coord),
443                None => self.disposition.ordinary_disposition(coord).0,
444            };
445            !matches!(
446                disposition,
447                crate::engine::FormulaReplayCoordinateDisposition::Direct
448                    | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
449            )
450        })
451    }
452}
453
454type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
455type StagedFormulaBatches = Vec<(String, StagedSheet)>;
456
457type CompressedReplayBatch = (
458    FormulaIngestBatch,
459    crate::engine::FormulaCompressedSourceBatch,
460);
461type PreparedSourceBatch = (
462    FormulaIngestBatch,
463    crate::engine::FormulaCompressedSourceReport,
464    crate::engine::FormulaCompressedPreparation,
465);
466type PreparedStagedFormulaBatches = (
467    PreparedFormulaBatches,
468    Vec<CompressedReplayBatch>,
469    Vec<PreparedSourceBatch>,
470    // Formulas whose planning can fail (see `formula_may_fail_planning`).
471    FxHashSet<crate::engine::arena::AstNodeId>,
472);
473
474/// Backend-neutral source-family ingress. Adapters may prepare candidates and
475/// submit exact replay; every formula is still materialized per cell.
476#[doc(hidden)]
477pub struct SourceFormulaIngress<'a, R> {
478    engine: &'a mut Engine<R>,
479}
480
481impl<R> SourceFormulaIngress<'_, R>
482where
483    R: EvaluationContext,
484{
485    pub fn prepare_families(
486        &mut self,
487        sheet_name: &str,
488        families: &[crate::engine::SourceFormulaFamily],
489    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
490        self.engine.observe_function_semantic_epoch()?;
491        Ok(self
492            .engine
493            .prepare_source_formula_families(sheet_name, families))
494    }
495
496    pub fn prepare_eager_proposals(
497        &mut self,
498        sheet_name: &str,
499        families: &[crate::engine::SourceFormulaFamily],
500        partitions: &[crate::engine::PartitionedSourceFormulaFamily],
501        formula_record_count: u64,
502        replay: Box<dyn crate::engine::DeferredFormulaReplay>,
503    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
504        self.engine.observe_function_semantic_epoch()?;
505        let replay = Arc::new(std::sync::Mutex::new(replay));
506        self.engine.prepare_source_formula_proposals(
507            sheet_name,
508            families,
509            partitions,
510            partitions,
511            formula_record_count,
512            replay,
513            &BTreeSet::new(),
514            &Default::default(),
515            None,
516        )
517    }
518
519    pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
520        self.engine.stage_deferred_formula_package(package);
521    }
522
523    pub fn ingest_replay_batches(
524        &mut self,
525        batches: Vec<(
526            FormulaIngestBatch,
527            crate::engine::FormulaCompressedSourceBatch,
528        )>,
529    ) -> Result<FormulaIngestReport, ExcelError> {
530        self.engine
531            .ingest_compressed_formula_source_batches(batches)
532    }
533
534    pub fn finish_prepared(
535        &mut self,
536        batches: Vec<(
537            FormulaIngestBatch,
538            crate::engine::FormulaCompressedSourceReport,
539            crate::engine::FormulaCompressedPreparation,
540        )>,
541    ) -> Result<FormulaIngestReport, ExcelError> {
542        self.engine.finish_compressed_formula_sources(batches)
543    }
544}
545
546// Computed-write coalescing pays a fixed grouping/planning cost. For very narrow
547// layers there is not enough work to amortize it, and the direct point-write path
548// is faster while preserving the same visibility semantics.
549const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
550
551/// Equal values, numbers by bits (debug oracles).
552#[cfg(debug_assertions)]
553fn same_value_bits(a: &LiteralValue, b: &LiteralValue) -> bool {
554    match (a, b) {
555        (LiteralValue::Number(x), LiteralValue::Number(y)) => x.to_bits() == y.to_bits(),
556        _ => a == b,
557    }
558}
559
560/// Whether a layer's computed writes are buffered: not for a chain unit,
561/// whose members read the ones written before them.
562fn buffer_layer_writes(layer: &crate::engine::scheduler::Layer) -> bool {
563    !layer.sequential && layer.vertices.len() >= COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH
564}
565/// Adaptive layer parallelism (see `evaluate_layer_parallel`): a layer runs
566/// sequentially until either it has run `PARALLEL_LAYER_PROBE` or the rest,
567/// at the rate so far, is estimated at `PARALLEL_LAYER_WORTH` or more; then
568/// the rest goes to the thread pool. Tuned on Enron first eval (probe 30 /
569/// 100 / 300 µs, worth 50 / 150 / 400 µs; 300 / 150 best, no workbook
570/// slower).
571/// Candidate count from which schedule preparation sorts on the pool.
572const PARALLEL_SCHEDULE_MIN_CANDIDATES: usize = 16 * 1024;
573/// Plan reuse: a schedule of more candidates than this can become the
574/// base, and a request takes the base restricted to it when the base is at
575/// most `BASE_SCHEDULE_RATIO` times the request.
576const BASE_SCHEDULE_MIN_VERTICES: usize = 64;
577const BASE_SCHEDULE_RATIO: usize = 256;
578/// A restricted schedule keeps the base's layers, which can split a family
579/// run the planner would keep whole (each piece then builds its own
580/// criteria index): below this many candidates planning costs less than
581/// that (real_ops_model: 75 candidates, 31 us to plan, +0.3 ms restricted).
582const BASE_SCHEDULE_MIN_REQUEST: usize = 128;
583const PARALLEL_LAYER_PROBE: std::time::Duration = std::time::Duration::from_micros(300);
584const PARALLEL_LAYER_WORTH: std::time::Duration = std::time::Duration::from_micros(150);
585/// A member this expensive (ns, measured by the probe) is its own task.
586const EXPENSIVE_VERTEX_NS: u128 = 10_000;
587/// [`Engine::live_cancellation_after_work`] messages.
588const UNIT_CANCELLED: &str = "Evaluation cancelled; the evaluated unit was not committed";
589const GROUP_CANCELLED: &str =
590    "Parallel evaluation cancelled; the evaluated group was not committed";
591const SCC_CANCELLED: &str = "Evaluation cancelled; the evaluated cycle was not completed";
592
593#[derive(Debug, Clone, PartialEq)]
594pub(crate) enum ComputedWrite {
595    Cell {
596        seq: u64,
597        sheet_id: SheetId,
598        row0: u32,
599        col0: u32,
600        value: OverlayValue,
601        format_id: Option<crate::format::FormatId>,
602    },
603    Rect {
604        seq: u64,
605        sheet_id: SheetId,
606        sr0: u32,
607        sc0: u32,
608        values: Vec<Vec<OverlayValue>>,
609    },
610    /// Consecutive rows `row0..` of one column (a family run's commit),
611    /// each with its format.
612    Run {
613        seq: u64,
614        sheet_id: SheetId,
615        row0: u32,
616        col0: u32,
617        entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
618    },
619}
620
621impl ComputedWrite {
622    #[inline]
623    pub(crate) fn seq(&self) -> u64 {
624        match self {
625            ComputedWrite::Cell { seq, .. }
626            | ComputedWrite::Rect { seq, .. }
627            | ComputedWrite::Run { seq, .. } => *seq,
628        }
629    }
630}
631
632#[derive(Debug, Default)]
633pub(crate) struct ComputedWriteBuffer {
634    writes: Vec<ComputedWrite>,
635    next_seq: u64,
636    estimated_bytes: usize,
637    formats_present: bool,
638}
639
640impl ComputedWriteBuffer {
641    const ENTRY_BASE_BYTES: usize = 32;
642
643    #[inline]
644    pub(crate) fn is_empty(&self) -> bool {
645        self.writes.is_empty()
646    }
647
648    #[inline]
649    pub(crate) fn len(&self) -> usize {
650        self.writes.len()
651    }
652
653    #[inline]
654    pub(crate) fn writes(&self) -> &[ComputedWrite] {
655        &self.writes
656    }
657
658    #[inline]
659    pub(crate) fn estimated_bytes(&self) -> usize {
660        self.estimated_bytes
661    }
662
663    #[cfg(feature = "tracing")]
664    fn traced_cell_count(&self) -> usize {
665        self.writes
666            .iter()
667            .map(|write| match write {
668                ComputedWrite::Cell { .. } => 1,
669                ComputedWrite::Rect { values, .. } => values.iter().map(Vec::len).sum(),
670                ComputedWrite::Run { entries, .. } => entries.len(),
671            })
672            .sum()
673    }
674
675    pub(crate) fn push_cell(
676        &mut self,
677        sheet_id: SheetId,
678        row0: u32,
679        col0: u32,
680        value: OverlayValue,
681    ) {
682        self.push_cell_with_format(sheet_id, row0, col0, value, None);
683    }
684
685    pub(crate) fn push_cell_with_format(
686        &mut self,
687        sheet_id: SheetId,
688        row0: u32,
689        col0: u32,
690        value: OverlayValue,
691        format_id: Option<crate::format::FormatId>,
692    ) {
693        let format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
694        self.formats_present |= format_id.is_some();
695        let seq = self.next_sequence();
696        self.estimated_bytes = self
697            .estimated_bytes
698            .saturating_add(Self::estimate_value_bytes(&value));
699        self.writes.push(ComputedWrite::Cell {
700            seq,
701            sheet_id,
702            row0,
703            col0,
704            value,
705            format_id,
706        });
707    }
708
709    pub(crate) fn push_column_run(
710        &mut self,
711        sheet_id: SheetId,
712        row0: u32,
713        col0: u32,
714        mut entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
715    ) {
716        let seq = self.next_sequence();
717        let mut added = 0usize;
718        for (value, format_id) in entries.iter_mut() {
719            *format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
720            self.formats_present |= format_id.is_some();
721            added = added.saturating_add(Self::estimate_value_bytes(value));
722        }
723        self.estimated_bytes = self.estimated_bytes.saturating_add(added);
724        self.writes.push(ComputedWrite::Run {
725            seq,
726            sheet_id,
727            row0,
728            col0,
729            entries,
730        });
731    }
732
733    pub(crate) fn push_rect(
734        &mut self,
735        sheet_id: SheetId,
736        sr0: u32,
737        sc0: u32,
738        values: Vec<Vec<OverlayValue>>,
739    ) {
740        let seq = self.next_sequence();
741        let added = values
742            .iter()
743            .flat_map(|row| row.iter())
744            .map(Self::estimate_value_bytes)
745            .fold(0usize, usize::saturating_add);
746        self.estimated_bytes = self.estimated_bytes.saturating_add(added);
747        self.writes.push(ComputedWrite::Rect {
748            seq,
749            sheet_id,
750            sr0,
751            sc0,
752            values,
753        });
754    }
755
756    pub(crate) fn clear(&mut self) {
757        self.writes.clear();
758        self.estimated_bytes = 0;
759        self.formats_present = false;
760    }
761
762    fn take_writes(&mut self) -> (Vec<ComputedWrite>, bool) {
763        self.estimated_bytes = 0;
764        let formats_present = std::mem::take(&mut self.formats_present);
765        (std::mem::take(&mut self.writes), formats_present)
766    }
767
768    fn next_sequence(&mut self) -> u64 {
769        let seq = self.next_seq;
770        self.next_seq = self.next_seq.wrapping_add(1);
771        seq
772    }
773
774    #[inline]
775    fn estimate_value_bytes(value: &OverlayValue) -> usize {
776        Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
777    }
778}
779
780#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
781struct ComputedWriteChunkKey {
782    sheet_id: SheetId,
783    col0: u32,
784    chunk_idx: usize,
785    chunk_start_row0: u32,
786}
787
788#[derive(Debug, Clone, PartialEq)]
789pub(crate) struct ComputedWriteChunkEntryPlan {
790    pub(crate) row_in_chunk: usize,
791    pub(crate) seq: u64,
792    pub(crate) value: OverlayValue,
793    pub(crate) format_id: Option<crate::format::FormatId>,
794}
795
796#[derive(Debug, Clone, PartialEq, Eq)]
797pub(crate) enum ComputedWriteChunkPlanShape {
798    Point,
799    SparseOffsets {
800        entries: usize,
801        span_len: usize,
802    },
803    DenseRange {
804        start: usize,
805        len: usize,
806    },
807    RunRange {
808        start: usize,
809        len: usize,
810        runs: usize,
811    },
812}
813
814#[derive(Debug, Clone, PartialEq, Eq)]
815pub(crate) enum ComputedWriteFormatClear {
816    Range { start: usize, end: usize },
817    Offsets(Vec<usize>),
818}
819
820#[derive(Debug, Clone, PartialEq)]
821pub(crate) enum ComputedWriteChunkFormatEffect {
822    NoFormatWork,
823    ClearStale(ComputedWriteFormatClear),
824    SetExplicit(Vec<(usize, Option<crate::format::FormatId>)>),
825}
826
827#[derive(Debug, Clone, PartialEq)]
828pub(crate) struct ComputedWriteChunkPlan {
829    pub(crate) sheet_id: SheetId,
830    pub(crate) col0: u32,
831    pub(crate) chunk_idx: usize,
832    pub(crate) chunk_start_row0: u32,
833    pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
834    pub(crate) shape: ComputedWriteChunkPlanShape,
835    pub(crate) format_effect: ComputedWriteChunkFormatEffect,
836}
837
838#[derive(Debug, Clone, Default, PartialEq)]
839pub(crate) struct ComputedWriteCoalescingPlan {
840    pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
841    pub(crate) input_cells: usize,
842    pub(crate) coalesced_cells: usize,
843    pub(crate) overwritten_cells: usize,
844}
845
846impl ComputedWriteCoalescingPlan {
847    #[inline]
848    pub(crate) fn is_empty(&self) -> bool {
849        self.chunks.is_empty()
850    }
851}
852
853impl ComputedWriteChunkPlan {
854    fn from_group(
855        key: ComputedWriteChunkKey,
856        mut entries: Vec<ComputedWriteChunkEntryPlan>,
857        formats_present: bool,
858        computed_lane_has_formats: bool,
859    ) -> (Self, usize) {
860        entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
861        let input_len = entries.len();
862        let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
863        for entry in entries {
864            if let Some(prev) = coalesced.last_mut()
865                && prev.row_in_chunk == entry.row_in_chunk
866            {
867                *prev = entry;
868                continue;
869            }
870            coalesced.push(entry);
871        }
872        let overwritten = input_len.saturating_sub(coalesced.len());
873        let shape = Self::classify_shape(&coalesced);
874        let format_effect = Self::classify_format_effect(
875            &coalesced,
876            &shape,
877            formats_present,
878            computed_lane_has_formats,
879        );
880        (
881            Self {
882                sheet_id: key.sheet_id,
883                col0: key.col0,
884                chunk_idx: key.chunk_idx,
885                chunk_start_row0: key.chunk_start_row0,
886                entries: coalesced,
887                shape,
888                format_effect,
889            },
890            overwritten,
891        )
892    }
893
894    fn classify_format_effect(
895        entries: &[ComputedWriteChunkEntryPlan],
896        shape: &ComputedWriteChunkPlanShape,
897        formats_present: bool,
898        computed_lane_has_formats: bool,
899    ) -> ComputedWriteChunkFormatEffect {
900        if !formats_present {
901            return if computed_lane_has_formats {
902                ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
903            } else {
904                ComputedWriteChunkFormatEffect::NoFormatWork
905            };
906        }
907
908        if entries.iter().all(|entry| entry.format_id.is_none()) {
909            return if computed_lane_has_formats {
910                ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
911            } else {
912                ComputedWriteChunkFormatEffect::NoFormatWork
913            };
914        }
915
916        ComputedWriteChunkFormatEffect::SetExplicit(
917            entries
918                .iter()
919                .map(|entry| (entry.row_in_chunk, entry.format_id))
920                .collect(),
921        )
922    }
923
924    fn format_clear_spec(
925        entries: &[ComputedWriteChunkEntryPlan],
926        shape: &ComputedWriteChunkPlanShape,
927    ) -> ComputedWriteFormatClear {
928        match shape {
929            ComputedWriteChunkPlanShape::DenseRange { start, len }
930            | ComputedWriteChunkPlanShape::RunRange { start, len, .. } => {
931                ComputedWriteFormatClear::Range {
932                    start: *start,
933                    end: start.saturating_add(*len),
934                }
935            }
936            ComputedWriteChunkPlanShape::Point
937            | ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
938                ComputedWriteFormatClear::Offsets(
939                    entries.iter().map(|entry| entry.row_in_chunk).collect(),
940                )
941            }
942        }
943    }
944
945    fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
946        debug_assert!(!entries.is_empty());
947        if entries.len() == 1 {
948            return ComputedWriteChunkPlanShape::Point;
949        }
950
951        let start = entries[0].row_in_chunk;
952        let end = entries[entries.len() - 1].row_in_chunk;
953        let span_len = end.saturating_sub(start).saturating_add(1);
954        if span_len != entries.len() {
955            return ComputedWriteChunkPlanShape::SparseOffsets {
956                entries: entries.len(),
957                span_len,
958            };
959        }
960
961        let runs = Self::run_count(entries);
962        if runs < entries.len() {
963            ComputedWriteChunkPlanShape::RunRange {
964                start,
965                len: entries.len(),
966                runs,
967            }
968        } else {
969            ComputedWriteChunkPlanShape::DenseRange {
970                start,
971                len: entries.len(),
972            }
973        }
974    }
975
976    fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
977        let mut runs = 0usize;
978        let mut prev: Option<&OverlayValue> = None;
979        for entry in entries {
980            if prev != Some(&entry.value) {
981                runs = runs.saturating_add(1);
982                prev = Some(&entry.value);
983            }
984        }
985        runs
986    }
987}
988
989#[cfg(feature = "tracing")]
990#[derive(Default)]
991struct TraceEvaluationCounters {
992    computed_vertices: usize,
993    cycles: usize,
994}
995
996pub struct Engine<R> {
997    criteria_mask_cache: criteria_mask_cache::CriteriaMaskCache,
998    pub(crate) graph: DependencyGraph,
999    resolver: R,
1000    pub(super) resolved_functions: crate::interpreter::ResolvedFunctions,
1001    resolved_sheet_names:
1002        FxHashMap<crate::engine::arena::StringId, (crate::SheetId, Option<usize>)>,
1003    resolution_sheet_revision: Option<(u64, u64, usize)>,
1004    resolution_arrow_sheet_names: Vec<Arc<str>>,
1005    resolved_arrow_sheets: Vec<Option<usize>>,
1006    /// Stable-ID index, invalidated before exposing arbitrary mutable storage.
1007    arrow_sheet_index: std::sync::OnceLock<Vec<Option<usize>>>,
1008    pub config: EvalConfig,
1009    workbook_load_limits: crate::engine::WorkbookLoadLimits,
1010    /// Clock for volatile date/time builtins, wrapped in a per-recalc
1011    /// snapshot: sampled once at the start of every evaluation request
1012    /// ([`Self::begin_evaluation_request`]) so all `NOW()`/`TODAY()` reads in
1013    /// one recalc — including SCC iteration passes — agree (spec §7.11).
1014    clock: crate::timezone::SnapshotClock,
1015    thread_pool: Option<Arc<rayon::ThreadPool>>,
1016    pub recalc_epoch: u64,
1017    snapshot_id: std::sync::atomic::AtomicU64,
1018    topology_epoch: u64,
1019    cached_static_schedule: Option<CachedScheduleEntry>,
1020    /// Program 3 (plan reuse): schedules of recent earlier requests, most
1021    /// recent last (a user alternating between a few inputs recalculates
1022    /// the same few closures). Bounded by `RECENT_SCHEDULES` entries and
1023    /// `RECENT_SCHEDULE_VERTICES` candidate vertices in total.
1024    recent_schedules: Vec<CachedScheduleEntry>,
1025    /// Program 3 (plan reuse): the largest current schedule seen (usually
1026    /// the first evaluation's). A request it covers, and that is not much
1027    /// smaller, takes its schedule restricted to the request instead of
1028    /// planning (`Schedule::restrict`).
1029    base_schedule: Option<CachedScheduleEntry>,
1030    #[cfg(any(test, feature = "benchmark_internal"))]
1031    recalc_reuse_probe: std::sync::Mutex<RecalcReuseProbe>,
1032    spill_mgr: ShimSpillManager,
1033    /// Arrow-backed storage for sheet values (Phase A)
1034    arrow_sheets: SheetStore,
1035    /// Workbook-local number-format registry.
1036    format_registry: crate::format::FormatRegistry,
1037    /// Derived formula formats keyed by grid position, never graph vertex identity.
1038    derived_formats: crate::engine::derived_formats::DerivedFormats,
1039    #[cfg(test)]
1040    derived_format_operations_for_test: std::sync::atomic::AtomicU64,
1041    #[cfg(test)]
1042    family_members_for_test: std::sync::atomic::AtomicU64,
1043    #[cfg(test)]
1044    invariant_bound_members_for_test: std::sync::atomic::AtomicU64,
1045    #[cfg(test)]
1046    lifted_members_for_test: std::sync::atomic::AtomicU64,
1047    #[cfg(test)]
1048    chained_members_for_test: std::sync::atomic::AtomicU64,
1049    #[cfg(test)]
1050    lane_clean_reads_for_test: std::sync::atomic::AtomicU64,
1051    #[cfg(test)]
1052    criteria_kernel_members_for_test: std::sync::atomic::AtomicU64,
1053    #[cfg(test)]
1054    memo_hits_for_test: std::sync::atomic::AtomicU64,
1055    /// Authority build last compressed (`maybe_compress_formulas`).
1056    compressed_at_build: Option<u64>,
1057    #[cfg(test)]
1058    computed_overlay_set_explicit_entry_operations_for_test: u64,
1059    #[cfg(test)]
1060    computed_overlay_stale_clear_range_effects_for_test: u64,
1061    #[cfg(test)]
1062    computed_overlay_stale_clear_offset_attempts_for_test: u64,
1063    #[cfg(test)]
1064    computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64,
1065    /// True if any edit after bulk load; disables Arrow reads for parity
1066    has_edited: bool,
1067    /// Overlay compaction counter (Phase C instrumentation)
1068    overlay_compactions: u64,
1069
1070    // Overlay memory observability / budget (ticket 503)
1071    computed_overlay_bytes_estimate: usize,
1072    computed_overlay_mirroring_disabled: bool,
1073    /// When true, RangeView resolution materializes from graph/Arrow base per-cell.
1074    /// This preserves correctness if we stop mirroring formula/spill outputs into computed overlays.
1075    pub(crate) force_materialize_range_views: bool,
1076    // Pass-scoped cache for Arrow used-row bounds per column
1077    row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
1078    // Snapshot-scoped final used-axis bounds for open-ended references.
1079    used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
1080    lookup_index_cache: LookupIndexCache,
1081    source_cache: Arc<std::sync::RwLock<SourceCache>>,
1082    /// Identity binding for opaque source-family preparations.
1083    source_formula_token: Arc<()>,
1084    /// Load-time staging of relocated formula copies (see `SourceFamilyMode`).
1085    pub(crate) source_family_mode: crate::engine::SourceFamilyMode,
1086    pub(crate) source_family_counters: crate::engine::SourceFamilyCounters,
1087    /// Dedicated identity binding for reusable recalculation plans.
1088    recalc_plan_token: Arc<()>,
1089    /// Staged formulas by sheet when `defer_graph_building` is enabled.
1090    staged_formulas: StagedFormulaMap,
1091    /// Presence and generation authority for ordinary staged formula discovery.
1092    staged_formula_index: StagedFormulaIndex,
1093    // Occupancy invalidation only: never a formula/read dependency.
1094    blocked_pending_spills: Vec<(VertexId, CellRef, Region)>,
1095    /// Anchors whose spill is blocked by a formula or by another spill's
1096    /// cell, keyed by that blocking cell: a change to the cell (an edit, or
1097    /// the other spill clearing) wakes them. Never a read dependency.
1098    spill_blocker_waiters: SpillBlockerWaiters,
1099    /// Per-sheet row visibility sidecar state.
1100    row_visibility: FxHashMap<SheetId, RowVisibilityState>,
1101    /// Cached row visibility masks keyed by sheet/span/mode/version.
1102    row_visibility_mask_cache: std::sync::RwLock<
1103        FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
1104    >,
1105    /// Non-fatal malformed formula diagnostics captured during ingest/graph-build.
1106    formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
1107    /// Last centralized formula ingest report.
1108    last_formula_ingest_report: Option<FormulaIngestReport>,
1109    /// Aggregate centralized formula ingest report for this engine.
1110    formula_ingest_report_total: FormulaIngestReport,
1111    /// Transient cancellation flag used during evaluation
1112    active_cancel_flag: Option<crate::engine::CancelToken>,
1113    /// Transient absolute deadline used by composed target and plan requests.
1114    active_evaluation_deadline: Option<Instant>,
1115
1116    /// Engine-level action depth.
1117    ///
1118    /// Ticket 614 introduces `Engine::action` as a stable, commit-only transaction surface.
1119    /// Nested actions are currently disallowed (deterministic rule) and will return an error.
1120    action_depth: u32,
1121
1122    // Phase 3b virtual-dependency convergence telemetry
1123    last_virtual_dep_telemetry: VirtualDepTelemetry,
1124    virtual_dep_fallback_activations: u64,
1125
1126    // Runtime-cycle SCC evaluation telemetry (RFC #112, Stage 2)
1127    last_cycle_telemetry: CycleTelemetry,
1128
1129    // C0 evaluation-resource observability. IDs are never reset or reused.
1130    next_evaluation_resource_request_id: u64,
1131    evaluation_resource_request_depth: usize,
1132    active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1133    last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1134    evaluation_resource_baseline: EvaluationResourceBaselineStats,
1135    evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1136    evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1137    evaluation_resource_config_diagnostic:
1138        Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1139    active_resource_ledger: Option<ResourceLedger>,
1140    source_cache_footprints: Vec<std::sync::Weak<std::sync::atomic::AtomicU64>>,
1141    source_cache_accounted: u64,
1142
1143    /// SCC members that entered iterative calculation (`CyclePolicy::Iterate`
1144    /// with a witnessed live cycle) during the current evaluation request
1145    /// and must re-run on the next one.
1146    ///
1147    /// Excel re-evaluates circular cells on EVERY recalc (the accumulator
1148    /// contract, spec §4/§7.6), but this engine's dirty model marks SCC
1149    /// members clean after a recalc and would otherwise skip them forever.
1150    /// Resolution: members of iterating SCCs are redirtied volatile-like at
1151    /// the end of the same recalc that iterated them
1152    /// ([`Self::redirty_for_next_recalc`], called wherever
1153    /// `redirty_volatiles` runs). The set is per-recalc, never persisted:
1154    /// if an edit breaks the cycle, the next recalc's SCC task either does
1155    /// not exist or settles as phantom, nothing re-registers, and the
1156    /// redirty chain stops by itself.
1157    ///
1158    /// SCCs that landed on an exact fixed point are exempt: they go to
1159    /// [`Self::retained_scc_members`] instead and stay clean until the dirty
1160    /// graph (or a config change) reaches them (#368).
1161    pending_iterative_redirty: Vec<VertexId>,
1162    /// Opt-in snapshot request: defer only next-cycle volatile redirty.
1163    snapshot_evaluation_active: bool,
1164    snapshot_volatile_redirty_pending: bool,
1165    /// Members of iterating SCCs retained across recalcs (#368), keyed to the
1166    /// id of the retained SCC they belong to (ids come from
1167    /// `next_retained_scc_id`; grouping is only used for telemetry).
1168    ///
1169    /// An SCC is retained when the recalc that iterated it stopped because
1170    /// every member reproduced its previous value exactly (|Δ| = 0, or
1171    /// identity for non-numeric members; never NaN-converged), before the
1172    /// `max_iterations` cap, with no volatile or dynamic-reference member.
1173    /// Such an SCC is a fixed point of its own inputs: running it again with
1174    /// the same inputs cannot change any value, so it is not redirtied. The
1175    /// dirty graph remains the validity authority — any edit that reaches a
1176    /// member dirties it like any other formula and the SCC task re-runs.
1177    /// Membership is dropped when a member runs in an SCC task again (it is
1178    /// then re-retained or re-registered for per-recalc redirty), when the
1179    /// vertex is deleted, or when
1180    /// [`Self::reconcile_retained_sccs_at_request_begin`] invalidates the
1181    /// whole set because a config knob outside the graph changed.
1182    retained_scc_members: FxHashMap<VertexId, u64>,
1183    next_retained_scc_id: u64,
1184    /// [`Self::retained_scc_config_fingerprint`] as of the last retention.
1185    /// `Engine::config` is a public field, so knobs that change a retained
1186    /// SCC's result (cycle policy/tolerance, date system, determinism,
1187    /// volatile seeding) can change between recalcs without touching the
1188    /// graph; a mismatch at request begin dirties every retained member.
1189    /// Only meaningful while `retained_scc_members` is non-empty.
1190    retained_scc_config_fingerprint: u64,
1191    /// Function-registry semantic epoch and runtime-provider revision as of
1192    /// the last time retained SCCs were reconciled against them. A newer
1193    /// epoch dirties only the retained members whose formula calls a changed
1194    /// function (or every member when the change log is incomplete).
1195    retained_scc_function_epoch_seen: u64,
1196    retained_scc_provider_revision_seen: Option<u64>,
1197    /// Retained members that were already dirty when the current request
1198    /// began, with the SCC id they carried. A member still holding that id
1199    /// at the end of the request was not touched by any SCC task — its
1200    /// cycle dissolved (it evaluated as an ordinary formula) or the request
1201    /// never reached it — so [`Self::redirty_for_next_recalc`] drops it from
1202    /// the retained set instead of letting it linger.
1203    retained_scc_dirty_at_begin: Vec<(VertexId, u64)>,
1204
1205    /// Final committed values of iterating-SCC members (spec §4 persistence).
1206    /// In canonical (value-cache disabled) mode the computed overlay is the
1207    /// ONLY home of a formula's value, and structural edits clear computed
1208    /// overlays wholesale (`clear_computed_overlay_after_row/_col`) —
1209    /// destroying iteration state (accumulators reset to 0; found by the
1210    /// iterate edge corpus). This snapshot lets the next SCC task re-seed
1211    /// members whose overlay entry vanished. Members registered for
1212    /// per-recalc redirty are refreshed by
1213    /// [`Self::redirty_for_next_recalc`]; retained members are written once
1214    /// when retained. Entries are dropped when the member's SCC task ends
1215    /// without iterating or when the vertex is deleted. Empty unless
1216    /// something iterated — zero cost otherwise.
1217    iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1218
1219    /// Global function-registry semantic epoch last observed.
1220    function_semantic_epoch_seen: u64,
1221    /// Runtime-provider semantic revision last observed.
1222    function_provider_revision_seen: Option<u64>,
1223
1224    #[cfg(feature = "tracing")]
1225    trace_evaluation_counters: TraceEvaluationCounters,
1226    #[cfg(test)]
1227    evaluation_request_begin_count_for_test: u64,
1228    #[cfg(any(test, feature = "test-support"))]
1229    before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1230    #[cfg(test)]
1231    before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1232    #[cfg(test)]
1233    before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1234    #[cfg(test)]
1235    inject_target_semantic_stale_once_for_test: bool,
1236    #[cfg(test)]
1237    force_virtual_dep_changes_remaining_for_test: usize,
1238    /// Dynamic-read freshness state (design §8.2).
1239    freshness: freshness::Freshness,
1240    #[cfg(test)]
1241    fail_evaluation_commit_preflight_once_for_test: bool,
1242    #[cfg(test)]
1243    target_preparation_fault_for_test:
1244        Option<crate::engine::target_preparation::TargetPreparationFault>,
1245    #[cfg(test)]
1246    force_non_cycle_schedule_fallback_for_test: bool,
1247    #[cfg(test)]
1248    before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1249    #[cfg(test)]
1250    after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1251}
1252
1253/// This wrapper is intentionally thin for ticket 614 (commit-only): it delegates to existing
1254/// `Engine` edit methods and does not create changelog boundaries or implement rollback.
1255impl<R: EvaluationContext> Engine<R> {
1256    pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1257        self.graph.ingest_pipeline(&self.resolver)
1258    }
1259}
1260
1261pub struct EngineAction<'a, R>
1262where
1263    R: EvaluationContext,
1264{
1265    engine: &'a mut Engine<R>,
1266    name: String,
1267    // Complete private mutation capture used by atomic actions.
1268    // Stored as a raw pointer to avoid creating aliasing `&mut` borrows alongside `&mut Engine`.
1269    capture: Option<*mut MutationCapture>,
1270    // Optional Arrow undo journal used by `Engine::action_atomic`.
1271    // Stored as a raw pointer to avoid aliasing issues with `&mut Engine`.
1272    arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1273    // True when this EngineAction must enforce conservative atomic transaction policy.
1274    atomic_policy: bool,
1275}
1276
1277impl<'a, R> EngineAction<'a, R>
1278where
1279    R: EvaluationContext,
1280{
1281    #[inline]
1282    fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1283        let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1284        let coord = crate::reference::Coord::from_excel(row, col, true, true);
1285        crate::reference::CellRef::new(sheet_id, coord)
1286    }
1287
1288    #[inline]
1289    pub fn name(&self) -> &str {
1290        &self.name
1291    }
1292
1293    #[inline]
1294    pub fn set_cell_value(
1295        &mut self,
1296        sheet: &str,
1297        row: u32,
1298        col: u32,
1299        value: LiteralValue,
1300    ) -> Result<(), crate::engine::EditorError> {
1301        if self.capture.is_some() {
1302            let old_value = self.engine.read_cell_value(sheet, row, col);
1303            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1304            let addr = self.addr_for(sheet, row, col);
1305            let Some(capture_ptr) = self.capture else {
1306                return Err(crate::engine::EditorError::TransactionFailed {
1307                    reason: "action_with_logger: missing mutation capture".to_string(),
1308                });
1309            };
1310
1311            // For atomic journal mode, record computed overlay effects for this cell.
1312            // Delta-overlay undo is recorded semantically based on old_value/old_formula.
1313            let old_comp = if self.arrow_undo.is_some() {
1314                self.engine.read_computed_overlay_cell(sheet, row, col)
1315            } else {
1316                None
1317            };
1318
1319            if self.engine.graph_admission_enabled() {
1320                let admission =
1321                    self.engine
1322                        .graph
1323                        .preview_value_mutation(addr.sheet_id, row, col)?;
1324                self.engine.preflight_graph_admission(admission)?;
1325            }
1326            if old_formula.is_none() {
1327                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1328            }
1329
1330            let delta_old_sem = if old_formula.is_some() {
1331                None
1332            } else {
1333                Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1334            };
1335
1336            let start_len = unsafe { (&*capture_ptr).len() };
1337
1338            // Safety: `capture_ptr` comes from a unique operation-local `&mut MutationCapture`.
1339            let capture = unsafe { &mut *capture_ptr };
1340            self.engine.edit_with_capture(capture, |editor| {
1341                editor.set_cell_value_with_old_state(
1342                    addr,
1343                    value.clone(),
1344                    old_value.clone(),
1345                    old_formula.clone(),
1346                );
1347            })?;
1348            self.engine.record_structural_change(StructuralScope::Cell {
1349                sheet: addr.sheet_id,
1350                row: addr.coord.row(),
1351                col: addr.coord.col(),
1352            });
1353
1354            if let Some(undo_ptr) = self.arrow_undo {
1355                // 1) Spill snapshot operations (computed overlay rect restore).
1356                let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1357                let undo = unsafe { &mut *undo_ptr };
1358                self.engine
1359                    .record_spill_ops_into_arrow_undo(undo, new_events);
1360
1361                // 2) Delta/computed overlay single-cell deltas.
1362                let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1363                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1364                let row0 = row.saturating_sub(1);
1365                let col0 = col.saturating_sub(1);
1366                let delta_new_sem = Some(value.clone());
1367                undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1368                undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1369            }
1370            Ok(())
1371        } else {
1372            self.engine
1373                .set_cell_value(sheet, row, col, value)
1374                .map_err(crate::engine::EditorError::from)
1375        }
1376    }
1377
1378    #[inline]
1379    pub fn set_cell_formula(
1380        &mut self,
1381        sheet: &str,
1382        row: u32,
1383        col: u32,
1384        ast: ASTNode,
1385    ) -> Result<(), crate::engine::EditorError> {
1386        if self.capture.is_some() {
1387            let old_value = self.engine.read_cell_value(sheet, row, col);
1388            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1389            let addr = self.addr_for(sheet, row, col);
1390            let Some(capture_ptr) = self.capture else {
1391                return Err(crate::engine::EditorError::TransactionFailed {
1392                    reason: "action_with_logger: missing mutation capture".to_string(),
1393                });
1394            };
1395
1396            let admitted_formula = if self.engine.graph_admission_enabled() {
1397                let placement =
1398                    CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1399                let ingested = self.engine.ingest_pipeline().ingest_formula(
1400                    FormulaAstInput::Tree(ast.clone()),
1401                    placement,
1402                    None,
1403                )?;
1404                let admission = self.engine.graph.preview_formula_mutations(&[(
1405                    addr.sheet_id,
1406                    row,
1407                    col,
1408                    ingested.dep_plan.clone(),
1409                )])?;
1410                self.engine.preflight_graph_admission(admission)?;
1411                Some((ingested.ast_id, ingested.dep_plan))
1412            } else {
1413                None
1414            };
1415            if old_formula.is_none() {
1416                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1417            }
1418            let delta_old = if self.arrow_undo.is_some() {
1419                if old_formula.is_some() {
1420                    None
1421                } else {
1422                    Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1423                }
1424            } else {
1425                None
1426            };
1427            let start_len = unsafe { (&*capture_ptr).len() };
1428
1429            // Safety: `capture_ptr` comes from a unique operation-local `&mut MutationCapture`.
1430            let capture = unsafe { &mut *capture_ptr };
1431            self.engine.edit_with_capture(capture, |editor| {
1432                if let Some((ast_id, plan)) = admitted_formula {
1433                    editor.set_cell_formula_with_prepared_plan(
1434                        addr,
1435                        ast.clone(),
1436                        old_value,
1437                        old_formula,
1438                        ast_id,
1439                        plan,
1440                    );
1441                } else {
1442                    editor.set_cell_formula_with_old_state(
1443                        addr,
1444                        ast.clone(),
1445                        old_value,
1446                        old_formula,
1447                    );
1448                }
1449            })?;
1450            self.engine.record_structural_change(StructuralScope::Cell {
1451                sheet: addr.sheet_id,
1452                row: addr.coord.row(),
1453                col: addr.coord.col(),
1454            });
1455
1456            if let Some(undo_ptr) = self.arrow_undo {
1457                let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1458                let undo = unsafe { &mut *undo_ptr };
1459                self.engine
1460                    .record_spill_ops_into_arrow_undo(undo, new_events);
1461                let delta_new: Option<LiteralValue> = None;
1462                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1463                let row0 = row.saturating_sub(1);
1464                let col0 = col.saturating_sub(1);
1465                undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1466            }
1467            Ok(())
1468        } else {
1469            self.engine
1470                .set_cell_formula(sheet, row, col, ast)
1471                .map_err(crate::engine::EditorError::from)
1472        }
1473    }
1474
1475    #[inline]
1476    pub fn set_row_hidden(
1477        &mut self,
1478        sheet: &str,
1479        row_1based: u32,
1480        hidden: bool,
1481        source: RowVisibilitySource,
1482    ) -> Result<(), crate::engine::EditorError> {
1483        if self.capture.is_some() {
1484            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1485            let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1486            let old_hidden = self
1487                .engine
1488                .row_visibility
1489                .get(&sheet_id)
1490                .map(|state| state.is_row_hidden(row0, Some(source)))
1491                .unwrap_or(false);
1492            if old_hidden == hidden {
1493                return Ok(());
1494            }
1495
1496            let _ = self
1497                .engine
1498                .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1499
1500            let Some(capture_ptr) = self.capture else {
1501                return Err(crate::engine::EditorError::TransactionFailed {
1502                    reason: "action_with_logger: missing mutation capture".to_string(),
1503                });
1504            };
1505            unsafe { &mut *capture_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1506                sheet_id,
1507                row0,
1508                source,
1509                old_hidden,
1510                new_hidden: hidden,
1511            });
1512
1513            Ok(())
1514        } else {
1515            self.engine
1516                .set_row_hidden(sheet, row_1based, hidden, source)
1517        }
1518    }
1519
1520    #[inline]
1521    pub fn set_rows_hidden(
1522        &mut self,
1523        sheet: &str,
1524        start_row_1based: u32,
1525        end_row_1based: u32,
1526        hidden: bool,
1527        source: RowVisibilitySource,
1528    ) -> Result<(), crate::engine::EditorError> {
1529        if self.capture.is_some() {
1530            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1531            let (start_row0, end_row0) =
1532                Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1533
1534            let Some(capture_ptr) = self.capture else {
1535                return Err(crate::engine::EditorError::TransactionFailed {
1536                    reason: "action_with_logger: missing mutation capture".to_string(),
1537                });
1538            };
1539            let capture = unsafe { &mut *capture_ptr };
1540
1541            for row0 in start_row0..=end_row0 {
1542                let old_hidden = self
1543                    .engine
1544                    .row_visibility
1545                    .get(&sheet_id)
1546                    .map(|state| state.is_row_hidden(row0, Some(source)))
1547                    .unwrap_or(false);
1548                if old_hidden == hidden {
1549                    continue;
1550                }
1551
1552                let _ = self
1553                    .engine
1554                    .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1555
1556                capture.record(crate::engine::ChangeEvent::SetRowVisibility {
1557                    sheet_id,
1558                    row0,
1559                    source,
1560                    old_hidden,
1561                    new_hidden: hidden,
1562                });
1563            }
1564
1565            Ok(())
1566        } else {
1567            self.engine
1568                .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1569        }
1570    }
1571
1572    #[inline]
1573    pub fn insert_rows(
1574        &mut self,
1575        sheet: &str,
1576        before: u32,
1577        count: u32,
1578    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1579        if count == 0 {
1580            return Ok(crate::engine::ShiftSummary::default());
1581        }
1582        if self.capture.is_some() {
1583            let Some(capture_ptr) = self.capture else {
1584                return Err(crate::engine::EditorError::TransactionFailed {
1585                    reason: "action_atomic: missing mutation capture".to_string(),
1586                });
1587            };
1588
1589            let before0 = before.saturating_sub(1);
1590            self.engine.check_insert_room(sheet, before0, count, true)?;
1591            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1592            let occupancy = self.engine.structural_row_occupancy(sheet, sheet_id);
1593            let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1594
1595            // Graph structural insert (logged) - no snapshot bump.
1596            let summary = {
1597                let capture = unsafe { &mut *capture_ptr };
1598                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1599                    Ok(crate::engine::ShiftSummary::default());
1600                self.engine.edit_with_capture(capture, |editor| {
1601                    editor.set_structural_occupancy(occupancy);
1602                    out = editor.insert_rows(sheet_id, before0, count);
1603                })?;
1604                out?
1605            };
1606
1607            // Arrow insert (truth) + undo op.
1608            self.engine.ensure_arrow_sheet(sheet);
1609            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1610                asheet.insert_rows(before0 as usize, count as usize);
1611            }
1612            self.engine
1613                .purge_derived_formats_after_row(sheet_id, before0);
1614            self.engine
1615                .shift_row_visibility_insert(sheet_id, before0, count);
1616            self.engine.mark_moved_formula_vertices_dirty(&summary);
1617            self.engine
1618                .clear_computed_overlay_after_row(sheet, before0 as usize);
1619            self.engine
1620                .record_structural_change(StructuralScope::Region(affected_region));
1621            if let Some(undo_ptr) = self.arrow_undo {
1622                unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1623            }
1624            Ok(summary)
1625        } else {
1626            self.engine.insert_rows(sheet, before, count)
1627        }
1628    }
1629
1630    #[inline]
1631    pub fn delete_rows(
1632        &mut self,
1633        sheet: &str,
1634        start: u32,
1635        count: u32,
1636    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1637        if count == 0 {
1638            return Ok(crate::engine::ShiftSummary::default());
1639        }
1640        if self.atomic_policy {
1641            return Err(crate::engine::EditorError::TransactionUnsupported {
1642                reason:
1643                    "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1644                        .to_string(),
1645            });
1646        }
1647        self.engine.delete_rows(sheet, start, count)
1648    }
1649
1650    #[inline]
1651    pub fn insert_columns(
1652        &mut self,
1653        sheet: &str,
1654        before: u32,
1655        count: u32,
1656    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1657        if count == 0 {
1658            return Ok(crate::engine::ShiftSummary::default());
1659        }
1660        if self.capture.is_some() {
1661            let Some(capture_ptr) = self.capture else {
1662                return Err(crate::engine::EditorError::TransactionFailed {
1663                    reason: "action_atomic: missing mutation capture".to_string(),
1664                });
1665            };
1666
1667            let before0 = before.saturating_sub(1);
1668            self.engine
1669                .check_insert_room(sheet, before0, count, false)?;
1670            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1671            let occupancy = self.engine.structural_column_occupancy();
1672            let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1673
1674            let summary = {
1675                let capture = unsafe { &mut *capture_ptr };
1676                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1677                    Ok(crate::engine::ShiftSummary::default());
1678                self.engine.edit_with_capture(capture, |editor| {
1679                    editor.set_structural_occupancy(occupancy);
1680                    out = editor.insert_columns(sheet_id, before0, count);
1681                })?;
1682                out?
1683            };
1684
1685            self.engine.ensure_arrow_sheet(sheet);
1686            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1687                asheet.insert_columns(before0 as usize, count as usize);
1688            }
1689            self.engine
1690                .purge_derived_formats_after_col(sheet_id, before0);
1691            self.engine.mark_moved_formula_vertices_dirty(&summary);
1692            self.engine
1693                .clear_computed_overlay_after_col(sheet, before0 as usize);
1694            self.engine
1695                .record_structural_change(StructuralScope::Region(affected_region));
1696            if let Some(undo_ptr) = self.arrow_undo {
1697                unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1698            }
1699            Ok(summary)
1700        } else {
1701            self.engine.insert_columns(sheet, before, count)
1702        }
1703    }
1704
1705    #[inline]
1706    pub fn delete_columns(
1707        &mut self,
1708        sheet: &str,
1709        start: u32,
1710        count: u32,
1711    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1712        if count == 0 {
1713            return Ok(crate::engine::ShiftSummary::default());
1714        }
1715        if self.atomic_policy {
1716            return Err(crate::engine::EditorError::TransactionUnsupported {
1717                reason:
1718                    "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1719                        .to_string(),
1720            });
1721        }
1722        self.engine.delete_columns(sheet, start, count)
1723    }
1724
1725    /// Start an action from within an action.
1726    ///
1727    /// Nested actions are currently disallowed (ticket 614), so this will return a
1728    /// `EditorError::TransactionFailed` while an outer action is active.
1729    #[inline]
1730    pub fn action<T>(
1731        &mut self,
1732        name: impl AsRef<str>,
1733        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1734    ) -> Result<T, crate::engine::EditorError> {
1735        self.engine.action(name, f)
1736    }
1737}
1738
1739struct ActionDepthGuard<'a, R> {
1740    engine: *mut Engine<R>,
1741    _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1742}
1743
1744impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1745    fn drop(&mut self) {
1746        // Safety: the guard is created from a unique `&mut Engine` borrow and lives no longer
1747        // than the surrounding `Engine::action` call.
1748        unsafe {
1749            let e = &mut *self.engine;
1750            e.action_depth = e.action_depth.saturating_sub(1);
1751        }
1752    }
1753}
1754
1755#[derive(Default)]
1756struct SourceCache {
1757    scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1758    tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1759    /// Materialized views of external range sources: one build per source
1760    /// version, shared by every formula that reads it.
1761    external_views: FxHashMap<(String, Option<u64>), RangeView<'static>>,
1762}
1763
1764#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1765struct VisibilityMaskCacheKey {
1766    sheet_id: SheetId,
1767    start_row0: u32,
1768    end_row0: u32,
1769    mode: VisibilityMaskMode,
1770    version: u64,
1771}
1772
1773#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1774enum StructuralScope {
1775    Cell { sheet: SheetId, row: u32, col: u32 },
1776    Region(Region),
1777    Sheet(SheetId),
1778    RemovedSheet(SheetId),
1779    OpaqueGlobal,
1780    AllSheets,
1781}
1782
1783/// Anchors blocked by a formula or another spill, indexed by the blocking
1784/// cell so that a single-cell change finds its waiters in O(1).
1785#[derive(Default)]
1786struct SpillBlockerWaiters {
1787    by_cell: FxHashMap<CellRef, Vec<VertexId>>,
1788    by_anchor: FxHashMap<VertexId, CellRef>,
1789}
1790
1791impl SpillBlockerWaiters {
1792    /// Largest region scanned cell by cell; a larger one scans the waiters.
1793    const REGION_PROBE_CELLS: u64 = 4096;
1794
1795    fn is_empty(&self) -> bool {
1796        self.by_anchor.is_empty()
1797    }
1798
1799    fn wait(&mut self, anchor: VertexId, blocker: CellRef) {
1800        match self.by_anchor.insert(anchor, blocker) {
1801            Some(old) if old == blocker => return,
1802            Some(old) => self.unlink(anchor, old),
1803            None => {}
1804        }
1805        self.by_cell.entry(blocker).or_default().push(anchor);
1806    }
1807
1808    fn forget(&mut self, anchor: VertexId) {
1809        if self.by_anchor.is_empty() {
1810            return;
1811        }
1812        if let Some(old) = self.by_anchor.remove(&anchor) {
1813            self.unlink(anchor, old);
1814        }
1815    }
1816
1817    fn unlink(&mut self, anchor: VertexId, cell: CellRef) {
1818        if let Some(waiters) = self.by_cell.get_mut(&cell) {
1819            waiters.retain(|&v| v != anchor);
1820            if waiters.is_empty() {
1821                self.by_cell.remove(&cell);
1822            }
1823        }
1824    }
1825
1826    /// Remove and return the anchors waiting on a cell inside `scope`.
1827    fn take_affected(&mut self, scope: StructuralScope) -> Vec<VertexId> {
1828        let cells: Vec<CellRef> = match scope {
1829            StructuralScope::Cell { sheet, row, col } => {
1830                match Coord::try_new(row, col, true, true) {
1831                    Ok(coord) if self.by_cell.contains_key(&CellRef::new(sheet, coord)) => {
1832                        vec![CellRef::new(sheet, coord)]
1833                    }
1834                    _ => Vec::new(),
1835                }
1836            }
1837            StructuralScope::Region(region) => {
1838                let (rows, cols) = region.axis_ranges();
1839                let (r0, r1) = rows.query_bounds();
1840                let (c0, c1) = cols.query_bounds();
1841                // Widen before `+ 1`: an unbounded axis (every structural edit
1842                // records `rows_from` / `cols_from`) spans 0..=u32::MAX, and
1843                // the u32 sum wrapped to an area of 0.
1844                let area = (u64::from(r1.saturating_sub(r0)) + 1)
1845                    .saturating_mul(u64::from(c1.saturating_sub(c0)) + 1);
1846                if area <= Self::REGION_PROBE_CELLS {
1847                    let sheet = region.sheet_id();
1848                    // A small region can still lie past the grid; no waiter
1849                    // is registered there, so those points are skipped.
1850                    (r0..=r1)
1851                        .flat_map(|r| (c0..=c1).map(move |c| (r, c)))
1852                        .filter_map(|(r, c)| Coord::try_new(r, c, true, true).ok())
1853                        .map(|coord| CellRef::new(sheet, coord))
1854                        .filter(|cell| self.by_cell.contains_key(cell))
1855                        .collect()
1856                } else {
1857                    self.by_cell
1858                        .keys()
1859                        .filter(|cell| {
1860                            region.intersects(&Region::point(
1861                                cell.sheet_id,
1862                                cell.coord.row(),
1863                                cell.coord.col(),
1864                            ))
1865                        })
1866                        .copied()
1867                        .collect()
1868                }
1869            }
1870            StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => self
1871                .by_cell
1872                .keys()
1873                .filter(|cell| cell.sheet_id == sheet)
1874                .copied()
1875                .collect(),
1876            StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => {
1877                self.by_cell.keys().copied().collect()
1878            }
1879        };
1880        let mut woken = Vec::new();
1881        for cell in cells {
1882            if let Some(waiters) = self.by_cell.remove(&cell) {
1883                for anchor in waiters {
1884                    self.by_anchor.remove(&anchor);
1885                    woken.push(anchor);
1886                }
1887            }
1888        }
1889        woken
1890    }
1891}
1892
1893#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
1894enum LoggedEditImpact {
1895    NoOp,
1896    DataOnly,
1897    Topology,
1898}
1899
1900#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1901enum LoggedEditDirection {
1902    Original,
1903    InverseReplay,
1904    ForwardReplay,
1905}
1906
1907#[derive(Clone, Copy)]
1908struct InvalidationBaseline {
1909    snapshot_id: u64,
1910    topology_epoch: u64,
1911}
1912
1913struct SourceCacheSession {
1914    cache: Arc<std::sync::RwLock<SourceCache>>,
1915}
1916
1917impl Drop for SourceCacheSession {
1918    fn drop(&mut self) {
1919        if let Ok(mut g) = self.cache.write() {
1920            *g = SourceCache::default();
1921        }
1922    }
1923}
1924
1925#[derive(Debug)]
1926#[non_exhaustive]
1927pub struct EvalResult {
1928    pub computed_vertices: usize,
1929    pub cycle_errors: usize,
1930    pub elapsed: std::time::Duration,
1931}
1932
1933#[derive(Clone, Debug, PartialEq, Eq)]
1934#[non_exhaustive]
1935pub struct TableMetadata {
1936    pub name: String,
1937    pub sheet: String,
1938    pub start_row: u32,
1939    pub start_col: u32,
1940    pub end_row: u32,
1941    pub end_col: u32,
1942    pub header_row: bool,
1943    pub headers: Vec<String>,
1944    pub totals_row: bool,
1945}
1946
1947/// Read-only engine counters used by benchmark/instrumentation tooling.
1948///
1949/// These counters are deliberately observational: collecting them must not mutate engine state or
1950/// alter formula evaluation semantics.
1951///
1952/// The `formula_plane_*` counters are always `0`: FormulaPlane spans were removed and the
1953/// dependency authority is the only runtime path. They are kept for source compatibility.
1954#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1955#[non_exhaustive]
1956pub struct EngineBaselineStats {
1957    pub graph_vertex_count: usize,
1958    pub graph_formula_vertex_count: usize,
1959    pub graph_edge_count: usize,
1960    pub dirty_vertex_count: usize,
1961    pub evaluation_vertex_count: usize,
1962    pub formula_ast_root_count: usize,
1963    pub formula_ast_node_count: usize,
1964    pub staged_formula_count: usize,
1965    pub formula_plane_active_span_count: usize,
1966    pub formula_plane_producer_result_entries: usize,
1967    pub formula_plane_consumer_read_entries: usize,
1968    pub formula_plane_mixed_topology_cache_builds: u64,
1969    pub formula_plane_mixed_topology_cache_hits: u64,
1970    pub formula_plane_mixed_topology_cache_overflows: u64,
1971    pub formula_plane_dirty_pending_events: usize,
1972    pub formula_plane_dirty_region_events_recorded: u64,
1973    pub formula_plane_dirty_span_region_events_recorded: u64,
1974    pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1975    pub formula_plane_dirty_global_invalidations: u64,
1976    pub formula_plane_structural_span_candidates: u64,
1977    pub formula_plane_cycle_member_span_demotions: u64,
1978    pub formula_plane_array_result_span_demotions: u64,
1979    /// Members of exactly converged iterative SCCs currently retained across
1980    /// recalcs (#368).
1981    pub retained_scc_members: usize,
1982}
1983
1984#[derive(Debug, Clone, Default)]
1985#[non_exhaustive]
1986pub struct VirtualDepTelemetry {
1987    pub candidate_vertices_total: usize,
1988    pub vdeps_vertices_total: usize,
1989    pub vdeps_edges_total: usize,
1990    pub builder_elapsed_ms_total: u128,
1991    pub schedule_virtual_passes: usize,
1992    pub schedule_static_passes: usize,
1993    pub schedule_cache_hits: usize,
1994    pub schedule_cache_misses: usize,
1995    pub reused_schedule_vertices_total: usize,
1996    pub replan_iterations: usize,
1997    pub changed_vdeps_total: usize,
1998    pub bailout_reason: Option<&'static str>,
1999    pub fallback_mode_activations: u64,
2000}
2001
2002/// Per-recalc telemetry for SCC evaluation under `CycleDetection::Runtime`
2003/// (spec `formualizer-cycle-semantics-spec.md` §10).
2004///
2005/// Collection is unconditional: SCC tasks are rare relative to ordinary
2006/// vertex evaluation and the counters are a handful of integer adds per
2007/// task, so no config flag gates them (unlike [`VirtualDepTelemetry`],
2008/// which pays per-schedule costs). Counters reset at the start of every
2009/// evaluation request.
2010#[derive(Debug, Clone, Default, PartialEq)]
2011#[non_exhaustive]
2012pub struct CycleTelemetry {
2013    /// SCC tasks executed (static SCCs that reached Runtime evaluation).
2014    pub static_sccs: usize,
2015    /// SCC tasks whose live subgraph was acyclic — values produced.
2016    pub phantom_sccs: usize,
2017    /// Distinct live cycles witnessed across all SCC tasks.
2018    pub live_cycles_witnessed: usize,
2019    /// Cells stamped `#CIRC!` by Runtime SCC tasks.
2020    pub circ_cells_stamped: usize,
2021    /// Evaluation sweeps over (subsets of) SCC members, totalled across tasks
2022    /// (pass 1 included).
2023    pub settle_passes_total: usize,
2024    /// Largest pass count any single SCC task needed.
2025    pub max_passes_single_scc: usize,
2026    /// SCC tasks that entered iterative calculation (`CyclePolicy::Iterate`
2027    /// with a witnessed live cycle). RFC #113, Stage 3.
2028    pub iterated_sccs: usize,
2029    /// Iterating SCC tasks that stopped because every member passed the
2030    /// spec-§6 convergence test.
2031    pub converged_sccs: usize,
2032    /// SCC tasks that stopped at a pass cap. Under `CyclePolicy::Iterate`
2033    /// this is the Excel `max_iterations` cap (NOT an error — last values
2034    /// are kept; includes the no-convergence-test `max_iterations: 1`
2035    /// contract). Under `CyclePolicy::Error` it is the defensive acyclic
2036    /// settle cap (|SCC| + 2), which only a bug can hit.
2037    pub capped_sccs: usize,
2038    /// Largest `|Δ|` observed in any member's final-pass convergence
2039    /// comparison across iterating SCC tasks (numeric-class members only).
2040    /// `0.0` when no comparison ran (e.g. `max_iterations: 1`).
2041    pub max_abs_delta_at_stop: f64,
2042    /// Identical-bit NaN vs NaN member comparisons that were treated as
2043    /// converged (spec §6 NaN rule).
2044    pub nan_converged: usize,
2045    /// Retained iterative SCCs (#368) that had no dirty member at request begin and were therefore not
2046    /// re-run: their last exact fixed point is served as-is. Counted at
2047    /// request begin, so a demand-driven request that never reaches a
2048    /// retained SCC still reports it as reused.
2049    pub reused_sccs: usize,
2050    /// Members of the SCCs counted in `reused_sccs`.
2051    pub reused_scc_members: usize,
2052    /// Total wall-clock time spent inside Runtime SCC tasks.
2053    pub elapsed_ms: u128,
2054}
2055
2056#[derive(Debug, Clone, Copy)]
2057struct ScheduleBuildMeta {
2058    candidate_vertices: usize,
2059    vdeps_vertices: usize,
2060    vdeps_edges: usize,
2061    builder_elapsed_ms: u128,
2062    used_virtual_schedule: bool,
2063    schedule_cache_hit: bool,
2064    schedule_cache_eligible: bool,
2065}
2066
2067#[cfg(any(test, feature = "benchmark_internal"))]
2068#[doc(hidden)]
2069#[derive(Debug, Clone, Default)]
2070pub struct RecalcReuseProbe {
2071    pub schedule_requests: usize,
2072    pub schedule_cache_hits: usize,
2073    pub schedule_cache_misses: usize,
2074    pub schedule_cache_ineligible: usize,
2075    pub schedule_builds: usize,
2076    /// Program 3 plan reuse: misses served by restricting the base schedule.
2077    pub schedule_base_restrictions: usize,
2078    pub schedule_shared_handles: usize,
2079    pub schedule_retained_bytes: usize,
2080    pub legacy_target_requests: usize,
2081    pub target_schedule_builds: usize,
2082    pub demand_builds: usize,
2083    pub demand_vertices: usize,
2084    pub demand_clean_formulas: usize,
2085    pub demand_explicit_edges: usize,
2086    pub demand_virtual_builder_calls: usize,
2087}
2088
2089#[cfg(any(test, feature = "benchmark_internal"))]
2090fn schedule_probe_retained_bytes(schedule: &crate::engine::Schedule) -> usize {
2091    fn vector_bytes<T>(values: &Vec<T>) -> usize {
2092        values.capacity() * std::mem::size_of::<T>()
2093    }
2094
2095    [
2096        vector_bytes(&schedule.units),
2097        vector_bytes(&schedule.layers),
2098        vector_bytes(&schedule.cycles),
2099    ]
2100    .into_iter()
2101    .chain(
2102        schedule
2103            .layers
2104            .iter()
2105            .map(|layer| vector_bytes(&layer.vertices)),
2106    )
2107    .chain(schedule.cycles.iter().map(vector_bytes))
2108    .sum()
2109}
2110
2111#[derive(Debug, Clone)]
2112struct CachedScheduleEntry {
2113    topology_epoch: u64,
2114    /// Authority `(store revision, rev.dyn)` the schedule was planned from
2115    /// (design §8.4; always 0 without `unified_authority`).
2116    authority_revision: (u64, u64),
2117    /// The request's vertex list as runs of consecutive ids (formula ids
2118    /// come in column runs, so a whole-workbook request is a few runs).
2119    candidate_vertices: VertexIdRuns,
2120    schedule: Arc<crate::engine::scheduler::Schedule>,
2121}
2122
2123/// A vertex list stored as `(first id, run length)` runs of consecutive
2124/// ids, in list order.
2125#[derive(Debug, Clone, Default)]
2126struct VertexIdRuns(Vec<(u32, u32)>);
2127
2128impl VertexIdRuns {
2129    fn from_slice(ids: &[VertexId]) -> Self {
2130        let mut runs: Vec<(u32, u32)> = Vec::new();
2131        for v in ids {
2132            match runs.last_mut() {
2133                Some((first, len)) if first.checked_add(*len) == Some(v.0) => *len += 1,
2134                _ => runs.push((v.0, 1)),
2135            }
2136        }
2137        runs.shrink_to_fit();
2138        Self(runs)
2139    }
2140
2141    fn equals(&self, ids: &[VertexId]) -> bool {
2142        let mut rest = ids;
2143        for &(first, len) in &self.0 {
2144            let len = len as usize;
2145            if rest.len() < len {
2146                return false;
2147            }
2148            let (head, tail) = rest.split_at(len);
2149            if head
2150                .iter()
2151                .enumerate()
2152                .any(|(i, v)| v.0 != first.wrapping_add(i as u32))
2153            {
2154                return false;
2155            }
2156            rest = tail;
2157        }
2158        rest.is_empty()
2159    }
2160
2161    fn heap_bytes(&self) -> usize {
2162        self.0.capacity() * std::mem::size_of::<(u32, u32)>()
2163    }
2164
2165    /// The number of ids.
2166    fn len(&self) -> usize {
2167        self.0.iter().map(|&(_, len)| len as usize).sum()
2168    }
2169}
2170
2171#[cfg(test)]
2172mod vertex_id_runs_tests {
2173    use super::{VertexId, VertexIdRuns};
2174
2175    #[test]
2176    fn runs_compare_like_the_list() {
2177        let ids = |v: &[u32]| v.iter().map(|&i| VertexId(i)).collect::<Vec<_>>();
2178        let list = ids(&[5, 6, 7, 2, 3, 9, 10, 10]);
2179        let runs = VertexIdRuns::from_slice(&list);
2180        assert_eq!(runs.0, vec![(5, 3), (2, 2), (9, 2), (10, 1)]);
2181        assert!(runs.equals(&list));
2182        assert!(!runs.equals(&list[..7]));
2183        assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 11])));
2184        assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 10, 11])));
2185        assert!(VertexIdRuns::from_slice(&[]).equals(&[]));
2186        assert!(!VertexIdRuns::from_slice(&[]).equals(&list));
2187        let edge = ids(&[u32::MAX - 1, u32::MAX, 0]);
2188        assert!(VertexIdRuns::from_slice(&edge).equals(&edge));
2189    }
2190}
2191
2192/// Uncacheable requests keep their schedule inline without a shared allocation.
2193enum EvaluationSchedule {
2194    Owned(crate::engine::scheduler::Schedule),
2195    Shared(Arc<crate::engine::scheduler::Schedule>),
2196}
2197
2198impl std::ops::Deref for EvaluationSchedule {
2199    type Target = crate::engine::scheduler::Schedule;
2200
2201    fn deref(&self) -> &Self::Target {
2202        match self {
2203            Self::Owned(schedule) => schedule,
2204            Self::Shared(schedule) => schedule,
2205        }
2206    }
2207}
2208
2209type ScheduleBuildOutput = (
2210    crate::engine::scheduler::Schedule,
2211    FxHashMap<VertexId, Vec<VertexId>>,
2212    ScheduleBuildMeta,
2213);
2214
2215type EvaluationScheduleBuildOutput = (
2216    EvaluationSchedule,
2217    FxHashMap<VertexId, Vec<VertexId>>,
2218    ScheduleBuildMeta,
2219);
2220
2221/// Opaque, revision-bound recalculation recipe.
2222#[derive(Debug)]
2223pub struct RecalcPlan {
2224    key: RecalcPlanKey,
2225    kind: RecalcPlanKind,
2226}
2227
2228#[derive(Debug)]
2229struct RecalcPlanKey {
2230    engine_token: Arc<()>,
2231    revisions: PlanningRevisionSnapshot,
2232}
2233
2234#[derive(Clone, Debug, PartialEq, Eq)]
2235struct PlanningRevisionSnapshot {
2236    engine_topology_epoch: u64,
2237    graph_topology_revision: u64,
2238    staged: u64,
2239    symbols: u64,
2240    semantic: u64,
2241    provider: Option<u64>,
2242    deterministic_mode: crate::engine::DeterministicMode,
2243    budgets: crate::engine::EvaluationBudgets,
2244}
2245
2246#[derive(Debug)]
2247enum RecalcPlanKind {
2248    CompatibilityFull {
2249        schedule: crate::engine::Schedule,
2250        has_dynamic_refs: bool,
2251    },
2252    Target {
2253        targets: Vec<crate::engine::EvaluationTarget>,
2254        scope: crate::engine::PrepareScope,
2255        topology: RecalcTopology,
2256        dynamic_policy: DynamicPlanPolicy,
2257    },
2258}
2259
2260#[derive(Debug)]
2261enum RecalcTopology {
2262    RunLocalRecipe,
2263    Workbook,
2264}
2265
2266#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2267enum DynamicPlanPolicy {
2268    BoundedTargetReplan,
2269}
2270
2271impl RecalcPlan {
2272    /// Returns the retained compatibility schedule depth. Target plans retain a
2273    /// run-local recipe rather than a schedule, so their layer count is zero.
2274    pub fn layer_count(&self) -> usize {
2275        match &self.kind {
2276            RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2277            RecalcPlanKind::Target { .. } => 0,
2278        }
2279    }
2280
2281    pub fn has_dynamic_refs(&self) -> bool {
2282        match &self.kind {
2283            RecalcPlanKind::CompatibilityFull {
2284                has_dynamic_refs, ..
2285            } => *has_dynamic_refs,
2286            RecalcPlanKind::Target { .. } => false,
2287        }
2288    }
2289
2290    #[cfg(test)]
2291    pub(crate) fn force_stale_reasons_for_test(
2292        &mut self,
2293        reasons: &[formualizer_common::PlanStaleReason],
2294    ) {
2295        use formualizer_common::PlanStaleReason;
2296        for reason in reasons {
2297            match reason {
2298                PlanStaleReason::Engine => {
2299                    self.key.engine_token = Arc::new(());
2300                }
2301                PlanStaleReason::Provider => {
2302                    self.key.revisions.provider = Some(
2303                        self.key
2304                            .revisions
2305                            .provider
2306                            .unwrap_or_default()
2307                            .wrapping_add(1),
2308                    );
2309                }
2310                PlanStaleReason::Semantic => {
2311                    self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2312                }
2313                PlanStaleReason::Budget => {
2314                    let current = self.key.revisions.budgets.work.max_work_units;
2315                    self.key.revisions.budgets.work.max_work_units =
2316                        Some(current.unwrap_or_default().wrapping_add(1));
2317                }
2318                PlanStaleReason::Staged => {
2319                    self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2320                }
2321                PlanStaleReason::Symbols => {
2322                    self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2323                }
2324                PlanStaleReason::Graph => {
2325                    self.key.revisions.graph_topology_revision =
2326                        self.key.revisions.graph_topology_revision.wrapping_add(1);
2327                }
2328                _ => {}
2329            }
2330        }
2331    }
2332}
2333
2334#[cfg(any(test, feature = "test-support"))]
2335pub(crate) mod criteria_mask_test_hooks {
2336    use std::cell::Cell;
2337
2338    thread_local! {
2339        static MASK_CALLS_ROWS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
2340        static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2341        static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2342    }
2343
2344    pub(crate) fn take_mask_work() -> (usize, usize) {
2345        MASK_CALLS_ROWS.with(|c| c.replace((0, 0)))
2346    }
2347
2348    pub(crate) fn note_mask(rows: usize) {
2349        MASK_CALLS_ROWS.with(|c| {
2350            let (calls, work) = c.get();
2351            c.set((calls + 1, work + rows));
2352        });
2353    }
2354
2355    pub fn reset_text_segment_counters() {
2356        TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2357        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2358    }
2359
2360    pub fn text_segment_counters() -> (usize, usize) {
2361        let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2362        let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2363        (a, b)
2364    }
2365
2366    pub(crate) fn inc_total() {
2367        TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2368    }
2369    pub(crate) fn inc_all_null() {
2370        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2371    }
2372}
2373
2374#[cfg(test)]
2375pub(crate) mod visibility_mask_test_hooks {
2376    use std::cell::Cell;
2377
2378    thread_local! {
2379        static HITS: Cell<usize> = const { Cell::new(0) };
2380        static MISSES: Cell<usize> = const { Cell::new(0) };
2381        static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2382    }
2383
2384    pub fn reset() {
2385        HITS.with(|c| c.set(0));
2386        MISSES.with(|c| c.set(0));
2387        EVICTIONS.with(|c| c.set(0));
2388    }
2389
2390    pub fn counters() -> (usize, usize, usize) {
2391        let hits = HITS.with(|c| c.get());
2392        let misses = MISSES.with(|c| c.get());
2393        let evictions = EVICTIONS.with(|c| c.get());
2394        (hits, misses, evictions)
2395    }
2396
2397    pub(crate) fn inc_hit() {
2398        HITS.with(|c| c.set(c.get() + 1));
2399    }
2400
2401    pub(crate) fn inc_miss() {
2402        MISSES.with(|c| c.set(c.get() + 1));
2403    }
2404
2405    pub(crate) fn inc_eviction() {
2406        EVICTIONS.with(|c| c.set(c.get() + 1));
2407    }
2408}
2409
2410fn is_numeric_text_equality(pred: &crate::args::CriteriaPredicate) -> bool {
2411    match pred {
2412        crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(text)) => {
2413            text.trim().parse::<f64>().is_ok_and(f64::is_finite)
2414        }
2415        _ => false,
2416    }
2417}
2418
2419/// Resolve the null (non-number) rows of an `=n` / `<>n` criteria mask over
2420/// one row segment: numeric text compares by value, every other non-number
2421/// cell (blank, boolean, error) is unequal. Segments without text stay
2422/// vectorized.
2423fn fill_numeric_equality_nulls(
2424    view: &RangeView<'_>,
2425    row_start: usize,
2426    row_len: usize,
2427    col_in_view: usize,
2428    mask: &arrow_array::BooleanArray,
2429    n: f64,
2430    ne: bool,
2431) -> Option<arrow_array::BooleanArray> {
2432    use arrow_array::Array as _;
2433    let texts = view.slice_lowered_text(row_start, row_len);
2434    let text = texts.get(col_in_view).and_then(|t| t.as_ref());
2435    match text {
2436        None => {
2437            if !ne {
2438                // Nulls are already "no match".
2439                return Some(mask.clone());
2440            }
2441            let nulls = arrow::compute::is_null(mask).ok()?;
2442            crate::compute_prelude::boolean::or_kleene(mask, &nulls).ok()
2443        }
2444        Some(text) => {
2445            if text.len() != mask.len() {
2446                return None;
2447            }
2448            let mut out = arrow_array::builder::BooleanBuilder::with_capacity(mask.len());
2449            for i in 0..mask.len() {
2450                if mask.is_valid(i) {
2451                    out.append_value(mask.value(i));
2452                } else if text.is_valid(i) {
2453                    let equal = crate::locale::Locale::invariant()
2454                        .parse_number_invariant(text.value(i))
2455                        .is_some_and(|x| (x - n).abs() < 1e-12);
2456                    out.append_value(equal != ne);
2457                } else {
2458                    out.append_value(ne);
2459                }
2460            }
2461            Some(out.finish())
2462        }
2463    }
2464}
2465
2466fn compute_criteria_mask(
2467    view: &RangeView<'_>,
2468    col_in_view: usize,
2469    pred: &crate::args::CriteriaPredicate,
2470) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2471    use crate::compute_prelude::{boolean, cmp, concat_arrays};
2472    use arrow::compute::kernels::comparison::{ilike, nilike};
2473    use arrow_array::{
2474        Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2475    };
2476
2477    // Helper: apply a numeric predicate to a single Float64Array chunk
2478    fn apply_numeric_pred(
2479        chunk: &Float64Array,
2480        pred: &crate::args::CriteriaPredicate,
2481    ) -> Option<BooleanArray> {
2482        match pred {
2483            crate::args::CriteriaPredicate::Gt(n) => {
2484                cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2485            }
2486            crate::args::CriteriaPredicate::Ge(n) => {
2487                cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2488            }
2489            crate::args::CriteriaPredicate::Lt(n) => {
2490                cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2491            }
2492            crate::args::CriteriaPredicate::Le(n) => {
2493                cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2494            }
2495            crate::args::CriteriaPredicate::Eq(v) => match v {
2496                formualizer_common::LiteralValue::Number(x) => {
2497                    cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2498                }
2499                formualizer_common::LiteralValue::Int(i) => {
2500                    cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2501                }
2502                _ => None,
2503            },
2504            crate::args::CriteriaPredicate::Ne(v) => match v {
2505                formualizer_common::LiteralValue::Number(x) => {
2506                    cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2507                }
2508                formualizer_common::LiteralValue::Int(i) => {
2509                    cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2510                }
2511                _ => None,
2512            },
2513            _ => None,
2514        }
2515    }
2516
2517    // Check if this is a numeric predicate that can be applied per-chunk
2518    let is_numeric_pred = matches!(
2519        pred,
2520        crate::args::CriteriaPredicate::Gt(_)
2521            | crate::args::CriteriaPredicate::Ge(_)
2522            | crate::args::CriteriaPredicate::Lt(_)
2523            | crate::args::CriteriaPredicate::Le(_)
2524            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2525            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2526            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2527            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2528    );
2529
2530    // OPTIMIZED PATH: For numeric predicates, apply per-chunk and concatenate boolean masks.
2531    // This avoids materializing the full numeric column (64-bit per element) and instead
2532    // concatenates boolean masks (1-bit per element) - a 64x memory reduction.
2533    if is_numeric_pred {
2534        let mut bool_parts: Vec<BooleanArray> = Vec::new();
2535        // `=n` / `<>n`: the number lane is null for blank, text, boolean and
2536        // error cells. Blanks, booleans and errors never equal a number;
2537        // numeric text does (`5` matches "5"), as in `criteria_match`.
2538        let number = |v: &formualizer_common::LiteralValue| match v {
2539            formualizer_common::LiteralValue::Number(n) => Some(*n),
2540            formualizer_common::LiteralValue::Int(i) => Some(*i as f64),
2541            _ => None,
2542        };
2543        let equality = match pred {
2544            crate::args::CriteriaPredicate::Eq(v) => number(v).map(|n| (n, false)),
2545            crate::args::CriteriaPredicate::Ne(v) => number(v).map(|n| (n, true)),
2546            _ => None,
2547        };
2548        for res in view.numbers_slices() {
2549            let (rs, rl, cols_seg) = res.ok()?;
2550            if col_in_view < cols_seg.len() {
2551                let chunk = cols_seg[col_in_view].as_ref();
2552                let mut mask = apply_numeric_pred(chunk, pred)?;
2553                if let Some((n, ne)) = equality
2554                    && mask.null_count() > 0
2555                {
2556                    mask = fill_numeric_equality_nulls(view, rs, rl, col_in_view, &mask, n, ne)?;
2557                }
2558                bool_parts.push(mask);
2559            }
2560        }
2561
2562        if bool_parts.is_empty() {
2563            return None;
2564        } else if bool_parts.len() == 1 {
2565            return Some(std::sync::Arc::new(bool_parts.remove(0)));
2566        } else {
2567            // Concatenate boolean masks (much cheaper than concatenating Float64 arrays)
2568            let anys: Vec<&dyn arrow_array::Array> = bool_parts
2569                .iter()
2570                .map(|a| a as &dyn arrow_array::Array)
2571                .collect();
2572            let conc: ArrayRef = concat_arrays(&anys).ok()?;
2573            let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2574            return Some(std::sync::Arc::new(ba));
2575        }
2576    }
2577
2578    // Wildcards and numeric text equality can match non-text cells. The lowered
2579    // base lane is text-only, unlike the scalar matcher. Keep the vectorized
2580    // path for text-only data, but cache a scalar-equivalent mask for mixed data.
2581    if is_numeric_text_equality(pred)
2582        || matches!(pred, crate::args::CriteriaPredicate::TextLike { .. })
2583    {
2584        for tags in view.type_tags_slices() {
2585            let (_, _, cols) = tags.ok()?;
2586            let tags = cols.get(col_in_view)?;
2587            if tags.values().iter().any(|tag| {
2588                *tag == crate::arrow_store::TypeTag::Empty as u8
2589                    || *tag == crate::arrow_store::TypeTag::Number as u8
2590                    || *tag == crate::arrow_store::TypeTag::Boolean as u8
2591            }) {
2592                let mut mask = BooleanBuilder::new();
2593                for chunk in view.iter_row_chunks() {
2594                    let chunk = chunk.ok()?;
2595                    for row in chunk.row_start..chunk.row_start + chunk.row_len {
2596                        mask.append_value(crate::builtins::criteria_match(
2597                            pred,
2598                            &view.get_cell(row, col_in_view),
2599                        ));
2600                    }
2601                }
2602                return Some(std::sync::Arc::new(mask.finish()));
2603            }
2604        }
2605    }
2606
2607    // Keep spreadsheet tilde escapes in the shared scalar matcher.
2608    if matches!(pred, crate::args::CriteriaPredicate::TextLike { pattern, .. }
2609        if pattern.contains('~'))
2610    {
2611        return None;
2612    }
2613
2614    // TEXT PATH: build masks per row-chunk using lowered text slices.
2615    // This avoids concatenating full-string columns just to compute a boolean mask.
2616    // `"="` / `"<>"`: truly blank cells only (empty text is not blank).
2617    if matches!(
2618        pred,
2619        crate::args::CriteriaPredicate::IsBlank | crate::args::CriteriaPredicate::NotBlank
2620    ) {
2621        let want_blank = matches!(pred, crate::args::CriteriaPredicate::IsBlank);
2622        let mut bool_parts: Vec<BooleanArray> = Vec::new();
2623        for tags in view.type_tags_slices() {
2624            let (_, _, cols) = tags.ok()?;
2625            let tags = cols.get(col_in_view)?;
2626            let mut bb = BooleanBuilder::with_capacity(tags.len());
2627            for i in 0..tags.len() {
2628                let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8;
2629                bb.append_value(blank == want_blank);
2630            }
2631            bool_parts.push(bb.finish());
2632        }
2633        return match bool_parts.len() {
2634            0 => None,
2635            1 => Some(std::sync::Arc::new(bool_parts.remove(0))),
2636            _ => {
2637                let anys: Vec<&dyn arrow_array::Array> = bool_parts
2638                    .iter()
2639                    .map(|a| a as &dyn arrow_array::Array)
2640                    .collect();
2641                let conc: ArrayRef = concat_arrays(&anys).ok()?;
2642                let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2643                Some(std::sync::Arc::new(ba))
2644            }
2645        };
2646    }
2647
2648    // Arrow LIKE uses backslash escapes; SQL punctuation is literal in Excel.
2649    let like_pattern = |text: &str, wildcards: bool| {
2650        let mut pattern = String::with_capacity(text.len());
2651        for ch in text.chars() {
2652            match ch {
2653                '%' | '_' | '\\' => {
2654                    pattern.push('\\');
2655                    pattern.push(ch);
2656                }
2657                '*' if wildcards => pattern.push('%'),
2658                '?' if wildcards => pattern.push('_'),
2659                _ => pattern.push(ch),
2660            }
2661        }
2662        pattern
2663    };
2664    let (text_kind, text_pat, empty_special) = match pred {
2665        crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2666            (0u8, like_pattern(&t.to_lowercase(), false), t.is_empty())
2667        }
2668        crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2669            (1u8, like_pattern(&t.to_lowercase(), false), false)
2670        }
2671        crate::args::CriteriaPredicate::TextLike {
2672            pattern,
2673            case_insensitive,
2674        } => {
2675            let p = if *case_insensitive {
2676                pattern.to_lowercase()
2677            } else {
2678                pattern.clone()
2679            };
2680            (2u8, like_pattern(&p, true), false)
2681        }
2682        _ => return None,
2683    };
2684
2685    let text_pat_is_empty = text_pat.is_empty();
2686    let ne_matches_blank = text_kind == 1 && !text_pat_is_empty;
2687    let pat = StringArray::new_scalar(text_pat);
2688    let mut bool_parts: Vec<BooleanArray> = Vec::new();
2689
2690    let mut tag_slices = view.type_tags_slices();
2691    for res in view.iter_row_chunks() {
2692        let cs = res.ok()?;
2693        if cs.row_len == 0 {
2694            continue;
2695        }
2696        #[cfg(test)]
2697        criteria_mask_test_hooks::inc_total();
2698
2699        let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2700        if col_in_view >= slices.len() {
2701            return None;
2702        }
2703
2704        let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2705        if empty_special || (text_kind == 1 && text_pat_is_empty) {
2706            let (tag_start, tag_len, tags) = tag_slices.next()?.ok()?;
2707            if tag_start != cs.row_start || tag_len != cs.row_len {
2708                return None;
2709            }
2710            let tags = tags.get(col_in_view)?;
2711            // A null text lane is not a blank cell: base numeric/boolean/error
2712            // cells also have null text. Consult the overlay-aware type tags,
2713            // and inspect strings only to distinguish empty text from text.
2714            let strings = seg_opt.and_then(|a| a.as_any().downcast_ref::<StringArray>());
2715            let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2716            for i in 0..cs.row_len {
2717                let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8
2718                    || (tags.value(i) == crate::arrow_store::TypeTag::Text as u8
2719                        && strings.is_some_and(|s| s.is_valid(i) && s.value(i).is_empty()));
2720                bb.append_value(if text_kind == 0 { blank } else { !blank });
2721            }
2722            #[cfg(test)]
2723            if seg_opt.is_none() {
2724                criteria_mask_test_hooks::inc_all_null();
2725            }
2726            bool_parts.push(bb.finish());
2727            continue;
2728        }
2729        let seg = match seg_opt {
2730            Some(s) => s,
2731            None => {
2732                #[cfg(test)]
2733                criteria_mask_test_hooks::inc_all_null();
2734                if (text_kind == 0 && empty_special) || ne_matches_blank {
2735                    // Eq("") treats nulls (Empty) as equal.
2736                    let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2737                    bb.append_n(cs.row_len, true);
2738                    bool_parts.push(bb.finish());
2739                } else {
2740                    // For non-empty patterns, ilike/nilike return null on null inputs.
2741                    bool_parts.push(BooleanArray::new_null(cs.row_len));
2742                }
2743                continue;
2744            }
2745        };
2746
2747        let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2748        let mut m = match text_kind {
2749            0 => ilike(seg_sa, &pat).ok()?,
2750            1 => nilike(seg_sa, &pat).ok()?,
2751            2 => ilike(seg_sa, &pat).ok()?,
2752            _ => return None,
2753        };
2754
2755        // Only fold blank/Empty (null) cells into the mask when the segment
2756        // actually contains any. The null-fill loop + or_kleene are pure
2757        // overhead on blank-free chunks, so a `<>text` (or `=""`) aggregation
2758        // over a column with no blanks stays fully vectorized on the ilike/
2759        // nilike result.
2760        if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2761            // Treat nulls as equal to empty string
2762            let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2763            for i in 0..seg_sa.len() {
2764                bb.append_value(seg_sa.is_null(i));
2765            }
2766            let nulls = bb.finish();
2767            m = boolean::or_kleene(&m, &nulls).ok()?;
2768        }
2769
2770        bool_parts.push(m);
2771    }
2772
2773    if bool_parts.is_empty() {
2774        None
2775    } else if bool_parts.len() == 1 {
2776        Some(std::sync::Arc::new(bool_parts.remove(0)))
2777    } else {
2778        let anys: Vec<&dyn arrow_array::Array> = bool_parts
2779            .iter()
2780            .map(|a| a as &dyn arrow_array::Array)
2781            .collect();
2782        let conc: ArrayRef = concat_arrays(&anys).ok()?;
2783        let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2784        Some(std::sync::Arc::new(ba))
2785    }
2786}
2787
2788#[derive(Debug, Clone)]
2789pub struct LayerInfo {
2790    pub vertex_count: usize,
2791    pub parallel_eligible: bool,
2792    pub sample_cells: Vec<String>, // Sample of up to 5 cell addresses
2793}
2794
2795#[derive(Debug, Clone)]
2796pub struct EvalPlan {
2797    pub total_vertices_to_evaluate: usize,
2798    pub layers: Vec<LayerInfo>,
2799    pub cycles_detected: usize,
2800    pub dirty_count: usize,
2801    pub volatile_count: usize,
2802    pub parallel_enabled: bool,
2803    pub estimated_parallel_layers: usize,
2804    pub target_cells: Vec<String>,
2805}
2806
2807/// Whether the configured FormulaPlane mode is ignored (spans never placed):
2808/// always, since the dependency authority is the runtime path (design §10).
2809#[inline]
2810fn plane_mode_ignored() -> bool {
2811    true
2812}
2813
2814/// Test-support probe for sibling-crate tests: true when span placement is
2815/// off because the FormulaPlane mode is ignored (see `plane_mode_ignored`).
2816#[cfg(feature = "test-support")]
2817#[doc(hidden)]
2818pub fn formula_plane_mode_ignored_for_test() -> bool {
2819    plane_mode_ignored()
2820}
2821
2822impl<R> Engine<R>
2823where
2824    R: EvaluationContext,
2825{
2826    /// # Panics
2827    /// Panics when `config.cycle` is invalid ([`CycleConfig::validate`],
2828    /// spec §2): `Iterate` with `detection: Static`, `max_iterations == 0`,
2829    /// or a negative/non-finite `max_change`. `EvalConfig::with_cycle`
2830    /// rejects these at build; this re-validates configs assembled via
2831    /// struct literals.
2832    pub fn new(resolver: R, config: EvalConfig) -> Self {
2833        // Under the unified authority the FormulaPlane mode is accepted and
2834        // ignored (design §10). Normalizing the stored mode keeps external
2835        // readers of `config` (loaders choosing a span-preparation route) on
2836        // the per-cell path; engine reads go through `formula_plane_mode()`.
2837        let config = if plane_mode_ignored() {
2838            EvalConfig {
2839                formula_plane_mode: FormulaPlaneMode::Off,
2840                ..config
2841            }
2842        } else {
2843            config
2844        };
2845        if let Err(msg) = config.cycle.validate() {
2846            panic!("invalid CycleConfig: {msg}");
2847        }
2848        crate::builtins::load_builtins();
2849        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2850            &config.evaluation_budgets,
2851            config.max_vertices,
2852            config.max_memory_mb,
2853            config.max_eval_time,
2854        );
2855
2856        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2857            #[cfg(feature = "system-clock")]
2858            {
2859                Arc::new(crate::timezone::SystemClock::new(
2860                    crate::timezone::TimeZoneSpec::default(),
2861                ))
2862            }
2863            #[cfg(not(feature = "system-clock"))]
2864            {
2865                Arc::new(crate::timezone::FixedClock::new(
2866                    chrono::DateTime::UNIX_EPOCH,
2867                    crate::timezone::TimeZoneSpec::Utc,
2868                ))
2869            }
2870        });
2871
2872        // Initialize thread pool based on config
2873        let thread_pool = if config.enable_parallel {
2874            let mut builder = ThreadPoolBuilder::new();
2875            if let Some(max_threads) = config.max_threads {
2876                builder = builder.num_threads(max_threads);
2877            }
2878
2879            match builder.build() {
2880                Ok(pool) => Some(Arc::new(pool)),
2881                Err(_) => {
2882                    // Fall back to sequential evaluation if thread pool creation fails
2883                    None
2884                }
2885            }
2886        } else {
2887            None
2888        };
2889
2890        // C1a retained/cache budgets are observational; cache defaults stay explicit.
2891        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2892        let function_provider_revision_seen = resolver.planning_semantic_revision();
2893        let mut engine = Self {
2894            criteria_mask_cache: Default::default(),
2895            graph: DependencyGraph::new_with_config(config.clone()),
2896            resolver,
2897            resolved_functions: Default::default(),
2898            resolved_sheet_names: FxHashMap::default(),
2899            resolution_sheet_revision: None,
2900            resolution_arrow_sheet_names: Vec::new(),
2901            resolved_arrow_sheets: Vec::new(),
2902            arrow_sheet_index: std::sync::OnceLock::new(),
2903            config,
2904            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2905            clock: crate::timezone::SnapshotClock::new(clock),
2906            thread_pool,
2907            recalc_epoch: 0,
2908            snapshot_id: std::sync::atomic::AtomicU64::new(1),
2909            topology_epoch: 0,
2910            cached_static_schedule: None,
2911            recent_schedules: Vec::new(),
2912            base_schedule: None,
2913            #[cfg(any(test, feature = "benchmark_internal"))]
2914            recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
2915            spill_mgr: ShimSpillManager::default(),
2916            arrow_sheets: SheetStore::default(),
2917            format_registry: crate::format::FormatRegistry::default(),
2918            derived_formats: Default::default(),
2919            #[cfg(test)]
2920            derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
2921            #[cfg(test)]
2922            family_members_for_test: std::sync::atomic::AtomicU64::new(0),
2923            #[cfg(test)]
2924            invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
2925            #[cfg(test)]
2926            lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
2927            #[cfg(test)]
2928            chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
2929            #[cfg(test)]
2930            lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
2931            #[cfg(test)]
2932            criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
2933            #[cfg(test)]
2934            memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
2935            compressed_at_build: None,
2936            #[cfg(test)]
2937            computed_overlay_set_explicit_entry_operations_for_test: 0,
2938            #[cfg(test)]
2939            computed_overlay_stale_clear_range_effects_for_test: 0,
2940            #[cfg(test)]
2941            computed_overlay_stale_clear_offset_attempts_for_test: 0,
2942            #[cfg(test)]
2943            computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
2944            has_edited: false,
2945            overlay_compactions: 0,
2946            computed_overlay_bytes_estimate: 0,
2947            computed_overlay_mirroring_disabled: false,
2948            force_materialize_range_views: false,
2949            row_bounds_cache: std::sync::RwLock::new(None),
2950            used_axis_bounds_cache: std::sync::RwLock::new(None),
2951            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2952            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2953            source_formula_token: Arc::new(()),
2954            source_family_mode: crate::engine::SourceFamilyMode::from_env(),
2955            source_family_counters: crate::engine::SourceFamilyCounters::default(),
2956            recalc_plan_token: Arc::new(()),
2957            staged_formulas: std::collections::HashMap::new(),
2958            staged_formula_index: StagedFormulaIndex::default(),
2959            blocked_pending_spills: Vec::new(),
2960            spill_blocker_waiters: SpillBlockerWaiters::default(),
2961            row_visibility: FxHashMap::default(),
2962            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2963            formula_parse_diagnostics: Vec::new(),
2964            last_formula_ingest_report: None,
2965            formula_ingest_report_total: FormulaIngestReport::default(),
2966            active_cancel_flag: None,
2967            active_evaluation_deadline: None,
2968            action_depth: 0,
2969            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2970            virtual_dep_fallback_activations: 0,
2971            last_cycle_telemetry: CycleTelemetry::default(),
2972            next_evaluation_resource_request_id: 1,
2973            evaluation_resource_request_depth: 0,
2974            active_evaluation_resource_request: None,
2975            last_evaluation_resource_request: None,
2976            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2977            evaluation_resource_request_started_at: None,
2978            evaluation_resource_budgets: resolved_resources.budgets,
2979            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2980            active_resource_ledger: None,
2981            source_cache_footprints: Vec::new(),
2982            source_cache_accounted: 0,
2983            pending_iterative_redirty: Vec::new(),
2984            snapshot_evaluation_active: false,
2985            snapshot_volatile_redirty_pending: false,
2986            retained_scc_members: FxHashMap::default(),
2987            next_retained_scc_id: 0,
2988            retained_scc_config_fingerprint: 0,
2989            retained_scc_function_epoch_seen: 0,
2990            retained_scc_provider_revision_seen: None,
2991            retained_scc_dirty_at_begin: Vec::new(),
2992            iterative_state_values: FxHashMap::default(),
2993            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2994            function_provider_revision_seen,
2995            #[cfg(feature = "tracing")]
2996            trace_evaluation_counters: TraceEvaluationCounters::default(),
2997            #[cfg(test)]
2998            evaluation_request_begin_count_for_test: 0,
2999            #[cfg(any(test, feature = "test-support"))]
3000            before_prepared_span_commit_hook: None,
3001            #[cfg(test)]
3002            before_target_preparation_commit_hook: None,
3003            #[cfg(test)]
3004            before_target_planning_snapshot_hook: None,
3005            #[cfg(test)]
3006            inject_target_semantic_stale_once_for_test: false,
3007            #[cfg(test)]
3008            force_virtual_dep_changes_remaining_for_test: 0,
3009            freshness: Default::default(),
3010            #[cfg(test)]
3011            fail_evaluation_commit_preflight_once_for_test: false,
3012            #[cfg(test)]
3013            target_preparation_fault_for_test: None,
3014            #[cfg(test)]
3015            force_non_cycle_schedule_fallback_for_test: false,
3016            #[cfg(test)]
3017            before_legacy_fallback_final_provider_sample_hook: None,
3018            #[cfg(test)]
3019            after_eager_proposal_commit_hook: None,
3020        };
3021        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
3022        engine.config.arrow_storage_enabled = true;
3023        engine.config.delta_overlay_enabled = true;
3024        engine.config.write_formula_overlay_enabled = true;
3025        let default_sheet = engine.graph.default_sheet_name().to_string();
3026        engine.ensure_arrow_sheet(&default_sheet);
3027        engine
3028    }
3029
3030    /// Create an Engine with a custom thread pool (for shared thread pool scenarios)
3031    ///
3032    /// # Panics
3033    /// Panics when `config.cycle` is invalid, exactly like [`Engine::new`].
3034    pub fn with_thread_pool(
3035        resolver: R,
3036        config: EvalConfig,
3037        thread_pool: Arc<rayon::ThreadPool>,
3038    ) -> Self {
3039        if let Err(msg) = config.cycle.validate() {
3040            panic!("invalid CycleConfig: {msg}");
3041        }
3042        crate::builtins::load_builtins();
3043        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
3044            &config.evaluation_budgets,
3045            config.max_vertices,
3046            config.max_memory_mb,
3047            config.max_eval_time,
3048        );
3049        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
3050            #[cfg(feature = "system-clock")]
3051            {
3052                Arc::new(crate::timezone::SystemClock::new(
3053                    crate::timezone::TimeZoneSpec::default(),
3054                ))
3055            }
3056            #[cfg(not(feature = "system-clock"))]
3057            {
3058                Arc::new(crate::timezone::FixedClock::new(
3059                    chrono::DateTime::UNIX_EPOCH,
3060                    crate::timezone::TimeZoneSpec::Utc,
3061                ))
3062            }
3063        });
3064        // C1a retained/cache budgets are observational; cache defaults stay explicit.
3065        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
3066        let function_provider_revision_seen = resolver.planning_semantic_revision();
3067        let mut engine = Self {
3068            criteria_mask_cache: Default::default(),
3069            graph: DependencyGraph::new_with_config(config.clone()),
3070            resolver,
3071            resolved_functions: Default::default(),
3072            resolved_sheet_names: FxHashMap::default(),
3073            resolution_sheet_revision: None,
3074            resolution_arrow_sheet_names: Vec::new(),
3075            resolved_arrow_sheets: Vec::new(),
3076            arrow_sheet_index: std::sync::OnceLock::new(),
3077            config,
3078            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
3079            clock: crate::timezone::SnapshotClock::new(clock),
3080            thread_pool: Some(thread_pool),
3081            recalc_epoch: 0,
3082            snapshot_id: std::sync::atomic::AtomicU64::new(1),
3083            topology_epoch: 0,
3084            cached_static_schedule: None,
3085            recent_schedules: Vec::new(),
3086            base_schedule: None,
3087            #[cfg(any(test, feature = "benchmark_internal"))]
3088            recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
3089            spill_mgr: ShimSpillManager::default(),
3090            arrow_sheets: SheetStore::default(),
3091            format_registry: crate::format::FormatRegistry::default(),
3092            derived_formats: Default::default(),
3093            #[cfg(test)]
3094            derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
3095            #[cfg(test)]
3096            family_members_for_test: std::sync::atomic::AtomicU64::new(0),
3097            #[cfg(test)]
3098            invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
3099            #[cfg(test)]
3100            lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
3101            #[cfg(test)]
3102            chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
3103            #[cfg(test)]
3104            lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
3105            #[cfg(test)]
3106            criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
3107            #[cfg(test)]
3108            memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
3109            compressed_at_build: None,
3110            #[cfg(test)]
3111            computed_overlay_set_explicit_entry_operations_for_test: 0,
3112            #[cfg(test)]
3113            computed_overlay_stale_clear_range_effects_for_test: 0,
3114            #[cfg(test)]
3115            computed_overlay_stale_clear_offset_attempts_for_test: 0,
3116            #[cfg(test)]
3117            computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
3118            has_edited: false,
3119            overlay_compactions: 0,
3120            computed_overlay_bytes_estimate: 0,
3121            computed_overlay_mirroring_disabled: false,
3122            force_materialize_range_views: false,
3123            row_bounds_cache: std::sync::RwLock::new(None),
3124            used_axis_bounds_cache: std::sync::RwLock::new(None),
3125            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
3126            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
3127            source_formula_token: Arc::new(()),
3128            source_family_mode: crate::engine::SourceFamilyMode::from_env(),
3129            source_family_counters: crate::engine::SourceFamilyCounters::default(),
3130            recalc_plan_token: Arc::new(()),
3131            staged_formulas: std::collections::HashMap::new(),
3132            staged_formula_index: StagedFormulaIndex::default(),
3133            blocked_pending_spills: Vec::new(),
3134            spill_blocker_waiters: SpillBlockerWaiters::default(),
3135            row_visibility: FxHashMap::default(),
3136            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
3137            formula_parse_diagnostics: Vec::new(),
3138            last_formula_ingest_report: None,
3139            formula_ingest_report_total: FormulaIngestReport::default(),
3140            active_cancel_flag: None,
3141            active_evaluation_deadline: None,
3142            action_depth: 0,
3143            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
3144            virtual_dep_fallback_activations: 0,
3145            last_cycle_telemetry: CycleTelemetry::default(),
3146            next_evaluation_resource_request_id: 1,
3147            evaluation_resource_request_depth: 0,
3148            active_evaluation_resource_request: None,
3149            last_evaluation_resource_request: None,
3150            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
3151            evaluation_resource_request_started_at: None,
3152            evaluation_resource_budgets: resolved_resources.budgets,
3153            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
3154            active_resource_ledger: None,
3155            source_cache_footprints: Vec::new(),
3156            source_cache_accounted: 0,
3157            pending_iterative_redirty: Vec::new(),
3158            snapshot_evaluation_active: false,
3159            snapshot_volatile_redirty_pending: false,
3160            retained_scc_members: FxHashMap::default(),
3161            next_retained_scc_id: 0,
3162            retained_scc_config_fingerprint: 0,
3163            retained_scc_function_epoch_seen: 0,
3164            retained_scc_provider_revision_seen: None,
3165            retained_scc_dirty_at_begin: Vec::new(),
3166            iterative_state_values: FxHashMap::default(),
3167            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
3168            function_provider_revision_seen,
3169            #[cfg(feature = "tracing")]
3170            trace_evaluation_counters: TraceEvaluationCounters::default(),
3171            #[cfg(test)]
3172            evaluation_request_begin_count_for_test: 0,
3173            #[cfg(any(test, feature = "test-support"))]
3174            before_prepared_span_commit_hook: None,
3175            #[cfg(test)]
3176            before_target_preparation_commit_hook: None,
3177            #[cfg(test)]
3178            before_target_planning_snapshot_hook: None,
3179            #[cfg(test)]
3180            inject_target_semantic_stale_once_for_test: false,
3181            #[cfg(test)]
3182            force_virtual_dep_changes_remaining_for_test: 0,
3183            freshness: Default::default(),
3184            #[cfg(test)]
3185            fail_evaluation_commit_preflight_once_for_test: false,
3186            #[cfg(test)]
3187            target_preparation_fault_for_test: None,
3188            #[cfg(test)]
3189            force_non_cycle_schedule_fallback_for_test: false,
3190            #[cfg(test)]
3191            before_legacy_fallback_final_provider_sample_hook: None,
3192            #[cfg(test)]
3193            after_eager_proposal_commit_hook: None,
3194        };
3195        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
3196        engine.config.arrow_storage_enabled = true;
3197        engine.config.delta_overlay_enabled = true;
3198        engine.config.write_formula_overlay_enabled = true;
3199        let default_sheet = engine.graph.default_sheet_name().to_string();
3200        engine.ensure_arrow_sheet(&default_sheet);
3201        engine
3202    }
3203
3204    pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
3205        &self.workbook_load_limits
3206    }
3207
3208    pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
3209        self.workbook_load_limits = limits;
3210    }
3211
3212    fn clear_source_cache(&self) {
3213        if let Ok(mut g) = self.source_cache.write() {
3214            *g = SourceCache::default();
3215        }
3216    }
3217
3218    pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
3219        &self.last_virtual_dep_telemetry
3220    }
3221
3222    /// Telemetry from Runtime SCC evaluation during the most recent
3223    /// evaluation request (always default-zero under `CycleDetection::Static`
3224    /// or when `enable_virtual_dep_telemetry` is off).
3225    pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
3226        &self.last_cycle_telemetry
3227    }
3228
3229    /// Resource observations for the most recently completed public evaluation request.
3230    pub fn last_evaluation_resource_request_stats(
3231        &self,
3232    ) -> Option<&EvaluationResourceRequestStats> {
3233        self.last_evaluation_resource_request.as_ref()
3234    }
3235
3236    /// Cumulative resource observations since engine creation or the last telemetry reset.
3237    pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
3238        self.evaluation_resource_baseline
3239    }
3240
3241    pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
3242        &self.evaluation_resource_budgets
3243    }
3244
3245    /// At most one diagnostic is emitted for deprecated resource fields.
3246    pub fn evaluation_resource_config_diagnostic(
3247        &self,
3248    ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
3249        self.evaluation_resource_config_diagnostic.as_ref()
3250    }
3251
3252    /// Reset accumulated and last-request observations without reusing request IDs.
3253    pub fn reset_evaluation_resource_telemetry(&mut self) {
3254        self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
3255        self.last_evaluation_resource_request = None;
3256    }
3257
3258    // Reconcile without replay locks: several packages may share the same Arc.
3259    // Weak tokens neither retain dead packages nor duplicate their allocations.
3260    fn reconcile_source_cache_footprints(&mut self) -> Result<(), ExcelError> {
3261        self.blocked_pending_spills.retain(|&(vertex, anchor, _)| {
3262            self.graph.vertex_exists(vertex)
3263                && self.graph.get_cell_ref(vertex) == Some(anchor)
3264                && matches!(
3265                    self.graph.get_vertex_kind(vertex),
3266                    VertexKind::FormulaScalar | VertexKind::FormulaArray
3267                )
3268        });
3269        if self.blocked_pending_spills.is_empty() {
3270            self.blocked_pending_spills = Vec::new();
3271        }
3272        let mut bytes = (self.blocked_pending_spills.capacity()
3273            * std::mem::size_of::<(VertexId, CellRef, Region)>()) as u64;
3274        self.source_cache_footprints.retain(|weak| {
3275            let Some(footprint) = weak.upgrade() else {
3276                return false;
3277            };
3278            bytes = bytes.saturating_add(footprint.load(std::sync::atomic::Ordering::Acquire));
3279            true
3280        });
3281        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3282            ledger
3283                .release_retained(self.source_cache_accounted)
3284                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3285            // Observe first: even a tightened budget cannot erase live ownership.
3286            ledger.observe_retained(bytes);
3287            self.source_cache_accounted = bytes;
3288            ledger
3289                .reserve_retained(0)
3290                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3291        }
3292        Ok(())
3293    }
3294
3295    fn duration_ns(duration: std::time::Duration) -> u64 {
3296        u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
3297    }
3298
3299    fn observe_evaluation_resource_request<T>(
3300        &mut self,
3301        kind: EvaluationRequestKind,
3302        evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3303    ) -> Result<T, ExcelError> {
3304        let outermost = self.evaluation_resource_request_depth == 0;
3305        if outermost && self.snapshot_volatile_redirty_pending {
3306            self.snapshot_volatile_redirty_pending = false;
3307            self.graph.redirty_volatiles();
3308        }
3309        #[cfg(feature = "tracing")]
3310        if outermost {
3311            self.trace_evaluation_counters = TraceEvaluationCounters::default();
3312        }
3313        #[cfg(feature = "tracing")]
3314        let request_kind = if matches!(
3315            kind,
3316            EvaluationRequestKind::Full
3317                | EvaluationRequestKind::FullWithDelta
3318                | EvaluationRequestKind::FullCancellable
3319                | EvaluationRequestKind::FullLogged
3320        ) {
3321            "full"
3322        } else {
3323            "targeted"
3324        };
3325        #[cfg(feature = "tracing")]
3326        let _request_span = outermost.then(|| {
3327            crate::engine::trace::fz_span!(
3328                tracing::Level::INFO,
3329                "evaluate",
3330                "evaluate.request",
3331                kind = request_kind,
3332                mode = ?FormulaPlaneMode::Off
3333            )
3334        });
3335        if outermost {
3336            let request_id = self.next_evaluation_resource_request_id;
3337            self.next_evaluation_resource_request_id = request_id
3338                .checked_add(1)
3339                .expect("evaluation resource request ID exhausted");
3340            self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
3341                request_id,
3342                kind,
3343                FormulaPlaneMode::Off,
3344                self.staged_formula_count(),
3345            ));
3346            self.evaluation_resource_baseline.record_started(request_id);
3347            self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
3348            self.source_cache_accounted = 0;
3349            self.active_resource_ledger = Some(ResourceLedger::new(
3350                Some(request_id),
3351                self.evaluation_resource_budgets.clone(),
3352            ));
3353        }
3354        self.evaluation_resource_request_depth =
3355            self.evaluation_resource_request_depth.saturating_add(1);
3356        let result = if outermost {
3357            self.reconcile_source_cache_footprints()
3358                .and_then(|()| self.resource_checkpoint(0))
3359                .and_then(|()| evaluate(self))
3360        } else {
3361            evaluate(self)
3362        };
3363        if outermost && result.is_err() {
3364            self.freshness_abort_pass();
3365        }
3366        self.evaluation_resource_request_depth =
3367            self.evaluation_resource_request_depth.saturating_sub(1);
3368
3369        let reconciliation = if outermost {
3370            self.reconcile_source_cache_footprints()
3371        } else {
3372            Ok(())
3373        };
3374        let result = result.and_then(|value| reconciliation.map(|()| value));
3375        if outermost {
3376            let total_ns = self
3377                .evaluation_resource_request_started_at
3378                .take()
3379                .map(|start| Self::duration_ns(start.elapsed()))
3380                .unwrap_or(0);
3381            let mut stats = self
3382                .active_evaluation_resource_request
3383                .take()
3384                .expect("outer evaluation resource request has active stats");
3385            let mut ledger = self
3386                .active_resource_ledger
3387                .take()
3388                .expect("outer evaluation resource request has active ledger");
3389            ledger.release_all_scratch();
3390            stats.ledger.update(ledger.snapshot());
3391            stats.outcome = match &result {
3392                Ok(_) => EvaluationRequestOutcome::Success,
3393                Err(error) if error.kind == ExcelErrorKind::Cancelled => {
3394                    EvaluationRequestOutcome::Cancelled
3395                }
3396                Err(_) => EvaluationRequestOutcome::Error,
3397            };
3398            if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
3399                stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
3400                    FormulaDirtyLeaseOutcome::RetainedOnCancellation
3401                } else {
3402                    FormulaDirtyLeaseOutcome::RetainedOnError
3403                };
3404            }
3405            stats.phases.total_ns = total_ns;
3406            let attributed = stats
3407                .phases
3408                .staged_prepare_ns
3409                .saturating_add(stats.phases.topology_ns)
3410                .saturating_add(stats.phases.materialization_ns);
3411            stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
3412            self.evaluation_resource_baseline.record_finished(&stats);
3413            self.last_evaluation_resource_request = Some(stats);
3414            crate::engine::trace::fz_event!(
3415                tracing::Level::INFO,
3416                "evaluate",
3417                "evaluate.summary",
3418                computed_vertices = self.trace_evaluation_counters.computed_vertices,
3419                cycles = self.trace_evaluation_counters.cycles,
3420                cancelled = matches!(
3421                    &result,
3422                    Err(error) if error.kind == ExcelErrorKind::Cancelled
3423                )
3424            );
3425        }
3426        result
3427    }
3428
3429    pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
3430        self.evaluation_resource_budgets = budgets.clone();
3431        self.config.evaluation_budgets = budgets.clone();
3432        self.graph.set_evaluation_budgets(budgets);
3433    }
3434
3435    #[cfg(test)]
3436    pub(crate) fn set_evaluation_budgets_for_test(
3437        &mut self,
3438        budgets: crate::engine::EvaluationBudgets,
3439    ) {
3440        self.set_evaluation_resource_budgets(budgets);
3441    }
3442
3443    fn preflight_evaluation_commit_window(
3444        &mut self,
3445        bounded_writes: usize,
3446    ) -> Result<crate::instant::FzInstant, ExcelError> {
3447        #[cfg(test)]
3448        if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
3449            return Err(crate::engine::ResourceLedgerError::Exhausted(
3450                formualizer_common::ResourceExhaustionDetail {
3451                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3452                    limit: 0,
3453                    observed: 1,
3454                    request_id: self
3455                        .active_evaluation_resource_request
3456                        .as_ref()
3457                        .map(|stats| stats.request_id),
3458                },
3459            )
3460            .into_excel_error());
3461        }
3462        let estimate = std::time::Duration::from_nanos(
3463            u64::try_from(bounded_writes)
3464                .unwrap_or(u64::MAX)
3465                .saturating_mul(100),
3466        );
3467        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3468            ledger
3469                .preflight_commit_window(estimate)
3470                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3471        }
3472        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3473            stats.evaluation_commit_preflight_count =
3474                stats.evaluation_commit_preflight_count.saturating_add(1);
3475            stats.evaluation_commit_estimated_ns = stats
3476                .evaluation_commit_estimated_ns
3477                .saturating_add(Self::duration_ns(estimate));
3478        }
3479        Ok(crate::instant::FzInstant::now())
3480    }
3481
3482    fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3483        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3484            stats.evaluation_commit_actual_ns = stats
3485                .evaluation_commit_actual_ns
3486                .saturating_add(Self::duration_ns(started.elapsed()));
3487        }
3488    }
3489
3490    /// Post-work cancellation boundary: call after evaluating a unit (or a
3491    /// parallel group) and before committing it. A function that observed
3492    /// the request's token returns `Err(Cancelled)`, which the evaluator
3493    /// turns into a `#CANCELLED` value; any other result computed across the
3494    /// signal is equally not a finished result. Neither may publish: the
3495    /// caller returns this error before committing, so the unit stays dirty,
3496    /// the request reports `Cancelled`, and a failed pass restores the
3497    /// vertices it already committed (`freshness_abort_pass`). The token is
3498    /// the discriminator: a `#CANCELLED` value with no live cancellation is
3499    /// ordinary data and commits. Deadlines are not checked here; finished
3500    /// work is not discarded on a deadline.
3501    fn live_cancellation_after_work(&self, message: &'static str) -> Result<(), ExcelError> {
3502        if self
3503            .active_cancel_flag
3504            .as_ref()
3505            .is_some_and(|cancel| cancel.is_cancelled())
3506        {
3507            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3508        }
3509        Ok(())
3510    }
3511
3512    fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3513        if self
3514            .active_cancel_flag
3515            .as_ref()
3516            .is_some_and(|cancel| cancel.is_cancelled())
3517        {
3518            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3519        }
3520        if self
3521            .active_evaluation_deadline
3522            .is_some_and(|deadline| Instant::now() >= deadline)
3523        {
3524            return Err(crate::engine::ResourceLedgerError::Exhausted(
3525                formualizer_common::ResourceExhaustionDetail {
3526                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3527                    limit: 0,
3528                    observed: 1,
3529                    request_id: self
3530                        .active_evaluation_resource_request
3531                        .as_ref()
3532                        .map(|request| request.request_id),
3533                },
3534            )
3535            .into_excel_error()
3536            .with_message(message));
3537        }
3538        Ok(())
3539    }
3540
3541    fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3542        let Some(ledger) = self.active_resource_ledger.as_mut() else {
3543            return Ok(());
3544        };
3545        ledger
3546            .charge_work(work_units)
3547            .and_then(|()| ledger.checkpoint_deadline())
3548            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3549    }
3550
3551    fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3552        if work_units == 0 {
3553            return self.resource_checkpoint(0);
3554        }
3555        while work_units > 0 {
3556            let chunk = work_units.min(256);
3557            self.resource_checkpoint(chunk)?;
3558            work_units -= chunk;
3559        }
3560        Ok(())
3561    }
3562
3563    fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3564        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3565            // Exact request topology and activation of the scratch cap are C1b. C1a records
3566            // scoped ownership but must not introduce a new skip or terminal path.
3567            ledger.observe_scratch(bytes);
3568        }
3569        Ok(())
3570    }
3571
3572    fn release_request_scratch(&mut self, bytes: u64) {
3573        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3574            let released = ledger.release_scratch(bytes);
3575            debug_assert!(
3576                released.is_ok(),
3577                "request scratch release exceeded the outstanding reservation"
3578            );
3579        }
3580    }
3581
3582    fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3583        self.active_resource_ledger
3584            .as_mut()
3585            .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3586            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3587    }
3588
3589    fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3590        self.active_resource_ledger
3591            .as_mut()
3592            .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3593            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3594    }
3595
3596    fn with_request_scratch<T>(
3597        &mut self,
3598        bytes: u64,
3599        work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3600    ) -> Result<T, ExcelError> {
3601        self.reserve_request_scratch(bytes)?;
3602        let result = work(self);
3603        self.release_request_scratch(bytes);
3604        result
3605    }
3606
3607    fn observe_staged_preparation(
3608        &mut self,
3609        selected: usize,
3610        retained: usize,
3611        elapsed: std::time::Duration,
3612    ) {
3613        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3614            stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3615            stats.staged_retained = retained as u64;
3616            stats.phases.staged_prepare_ns = stats
3617                .phases
3618                .staged_prepare_ns
3619                .saturating_add(Self::duration_ns(elapsed));
3620        }
3621    }
3622
3623    fn graph_admission_enabled(&self) -> bool {
3624        crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3625    }
3626
3627    fn preflight_graph_admission(
3628        &mut self,
3629        usage: crate::engine::resource_ledger::GraphAdmission,
3630    ) -> Result<(), ExcelError> {
3631        let request_id = self
3632            .active_evaluation_resource_request
3633            .as_ref()
3634            .map(|stats| stats.request_id);
3635        crate::engine::resource_ledger::preflight_graph_admission(
3636            &self.evaluation_resource_budgets,
3637            usage,
3638            request_id,
3639        )
3640        .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3641    }
3642
3643    fn prepared_legacy_admission(
3644        &mut self,
3645        plan: &PreparedLegacyGraphPlan,
3646        materialization_cells: u64,
3647    ) -> Result<(), ExcelError> {
3648        if !self.graph_admission_enabled() {
3649            return Ok(());
3650        }
3651        let stats = self.graph.baseline_stats();
3652        let added_edges = plan.planned_edge_count().ok_or_else(|| {
3653            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3654        })?;
3655        let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3656            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3657        })?;
3658        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3659            final_vertices: stats
3660                .graph_vertex_count
3661                .checked_add(plan.new_vertex_count())
3662                .ok_or_else(|| {
3663                    ExcelError::new(ExcelErrorKind::NImpl)
3664                        .with_message("graph vertex count overflow")
3665                })?,
3666            final_edges: stats
3667                .graph_edge_count
3668                .checked_sub(removed_edges)
3669                .and_then(|count| count.checked_add(added_edges))
3670                .ok_or_else(|| {
3671                    ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3672                })?,
3673            materialization_cells,
3674            added_vertices: plan.new_vertex_count(),
3675            added_edges,
3676        })
3677    }
3678
3679    fn observe_target_admission_failure(
3680        &mut self,
3681        reason: formualizer_common::ResourceExhaustionReason,
3682    ) {
3683        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3684            stats.target_admission_failure = Some(reason);
3685        }
3686    }
3687
3688    fn observe_target_preparation_report(
3689        &mut self,
3690        report: &crate::engine::PreparedTargetGraphReport,
3691    ) {
3692        let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3693            let index = match reason {
3694                crate::engine::OpaqueReason::DynamicReference => 0,
3695                crate::engine::OpaqueReason::RuntimeTextReference => 1,
3696                crate::engine::OpaqueReason::UnknownFunction => 2,
3697                crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3698                crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3699                crate::engine::OpaqueReason::UnresolvedName => 5,
3700                crate::engine::OpaqueReason::UnresolvedTable => 6,
3701                crate::engine::OpaqueReason::FormulaName => 7,
3702                crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3703                crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3704                crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3705            };
3706            1u64 << index
3707        };
3708        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3709            stats.staged_selected = report.selected_staged_cells as u64;
3710            stats.staged_retained = report.retained_staged_cells as u64;
3711            stats.target_requested = report.requested_targets as u64;
3712            stats.target_normalized_regions = report.normalized_regions as u64;
3713            stats.target_scope_level = match &report.widened_scope {
3714                crate::engine::PrepareScope::Exact => 0,
3715                crate::engine::PrepareScope::Sheets(_) => 1,
3716                crate::engine::PrepareScope::Workbook => 2,
3717            };
3718            stats.target_widening_reason_bits = report
3719                .widening_reasons
3720                .iter()
3721                .copied()
3722                .fold(0, |bits, reason| bits | reason_bit(reason));
3723            stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3724            stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3725            stats.target_commit_estimated_work = report.estimated_commit_work;
3726            stats.target_commit_actual_work = report.actual_commit_work;
3727            stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3728            stats.phases.staged_prepare_ns = stats
3729                .phases
3730                .staged_prepare_ns
3731                .saturating_add(Self::duration_ns(report.commit_window));
3732        }
3733    }
3734
3735    /// Begin a new evaluation request: reset per-recalc cycle telemetry and
3736    /// take the per-recalc volatile clock sample. Called at the start of
3737    /// every evaluation request that walks schedule units.
3738    /// The per-request function table when it describes the current registry,
3739    /// provider and arena; `None` outside a request when it may be stale.
3740    pub(crate) fn current_resolved_functions(
3741        &self,
3742    ) -> Option<&crate::interpreter::ResolvedFunctions> {
3743        self.resolved_functions
3744            .is_current(self.graph.data_store(), &self.resolver)
3745            .then_some(&self.resolved_functions)
3746    }
3747
3748    fn refresh_resolution_snapshots(&mut self) {
3749        self.resolved_functions
3750            .refresh(self.graph.data_store(), &self.resolver);
3751        let named_asts = self
3752            .graph
3753            .named_ranges_iter()
3754            .map(|(_, named)| named)
3755            .chain(self.graph.sheet_named_ranges_iter().map(|(_, named)| named))
3756            .filter_map(|named| match &named.definition {
3757                NamedDefinition::Formula { ast, .. } => Some(ast),
3758                _ => None,
3759            });
3760        self.resolved_functions.refresh_ast_functions(
3761            self.graph.data_store(),
3762            &self.resolver,
3763            named_asts,
3764            self.graph.symbol_revision(),
3765        );
3766        // Arrow sheet storage is publicly mutable. Compare names and order
3767        // exactly (including same-length replacements) without allocating on
3768        // unchanged requests; a fingerprint alone could silently collide.
3769        let revision = (
3770            self.graph.sheet_reg().revision(),
3771            self.graph.data_store().resolution_revision(),
3772            self.graph.data_store().sheet_names().count(),
3773        );
3774        let arrow_names_unchanged = self
3775            .resolution_arrow_sheet_names
3776            .iter()
3777            .map(|name| name.as_ref())
3778            .eq(self
3779                .arrow_sheets
3780                .sheets
3781                .iter()
3782                .map(|sheet| sheet.name.as_ref()));
3783        if self.resolution_sheet_revision == Some(revision) && arrow_names_unchanged {
3784            return;
3785        }
3786        self.resolution_arrow_sheet_names.clear();
3787        self.resolution_arrow_sheet_names.extend(
3788            self.arrow_sheets
3789                .sheets
3790                .iter()
3791                .map(|sheet| Arc::clone(&sheet.name)),
3792        );
3793        self.resolved_sheet_names.clear();
3794        for id in self.graph.data_store().sheet_names() {
3795            let name = self.graph.data_store().resolve_ast_string(id);
3796            if let Some(sheet_id) = self.graph.sheet_id(name) {
3797                self.resolved_sheet_names.insert(
3798                    id,
3799                    (
3800                        sheet_id,
3801                        self.arrow_sheets
3802                            .sheets
3803                            .iter()
3804                            .position(|sheet| sheet.name.as_ref() == name),
3805                    ),
3806                );
3807            }
3808        }
3809        self.resolved_arrow_sheets.clear();
3810        for (index, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
3811            if let Some(id) = self.graph.sheet_id(&sheet.name) {
3812                let len = usize::from(id).saturating_add(1);
3813                if self.resolved_arrow_sheets.len() < len {
3814                    self.resolved_arrow_sheets.resize(len, None);
3815                }
3816                if let Some(slot) = self.resolved_arrow_sheets.get_mut(usize::from(id)) {
3817                    *slot = Some(index);
3818                }
3819            }
3820        }
3821        self.resolution_sheet_revision = Some(revision);
3822    }
3823
3824    fn begin_evaluation_request(&mut self) {
3825        self.refresh_resolution_snapshots();
3826        self.freshness_begin_request();
3827        #[cfg(test)]
3828        {
3829            self.evaluation_request_begin_count_for_test = self
3830                .evaluation_request_begin_count_for_test
3831                .saturating_add(1);
3832        }
3833        self.last_cycle_telemetry = CycleTelemetry::default();
3834        self.graph.authority_sync();
3835        // Defensive: consumed at the end of the previous request; a request
3836        // that errored out mid-walk must not leak its members into this one.
3837        self.pending_iterative_redirty.clear();
3838        self.reconcile_retained_sccs_at_request_begin();
3839        // Spec §7.11: NOW()/TODAY() sample the clock ONCE per recalc; every
3840        // read within this request (including SCC iteration passes) observes
3841        // this sample.
3842        self.clock.refresh();
3843    }
3844
3845    /// End-of-recalc redirty: volatile vertices (as always) plus members of
3846    /// SCCs that iterated this recalc without reaching a retainable fixed
3847    /// point (`CyclePolicy::Iterate`), so circular cells re-evaluate on every
3848    /// recalc exactly like Excel's iterative calculation (spec §4
3849    /// persistence / §7.6 accumulator / §7.11 volatile redirty). Retained
3850    /// SCCs (`retained_scc_members`, #368) are left clean. Replaces the bare
3851    /// `graph.redirty_volatiles()` call at every evaluation-flow exit; must
3852    /// run AFTER the flow's `clear_dirty_flags`.
3853    fn redirty_for_next_recalc(&mut self) {
3854        if self.snapshot_evaluation_active {
3855            self.snapshot_volatile_redirty_pending = true;
3856        } else {
3857            self.graph.redirty_volatiles();
3858        }
3859        let pending = std::mem::take(&mut self.pending_iterative_redirty);
3860        let dirty_at_begin = std::mem::take(&mut self.retained_scc_dirty_at_begin);
3861        for (vertex, scc) in dirty_at_begin {
3862            if self.retained_scc_members.get(&vertex) == Some(&scc) {
3863                // No SCC task claimed this member during the request: the
3864                // cycle dissolved and it evaluated as an ordinary formula,
3865                // or the request never reached it. Either way it is no
3866                // longer a retained fixed point. Its persisted value is
3867                // obsolete only if it actually re-evaluated (clean now).
3868                self.retained_scc_members.remove(&vertex);
3869                if !self.graph.is_dirty(vertex) {
3870                    self.iterative_state_values.remove(&vertex);
3871                }
3872            }
3873        }
3874        if !self.iterative_state_values.is_empty() || !self.retained_scc_members.is_empty() {
3875            let graph = &self.graph;
3876            self.iterative_state_values
3877                .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3878            self.retained_scc_members
3879                .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3880        }
3881        // Refresh the §4-persistence snapshot for members that re-run each
3882        // recalc: these final values survive structural edits that clear the
3883        // computed overlay (the only value home in canonical mode) so the
3884        // next SCC task can re-seed from them (see `iterative_state_values`).
3885        for &vertex in &pending {
3886            if !self.graph.is_live_formula_vertex(vertex) {
3887                continue;
3888            }
3889            if let Some(cell) = self.graph.get_cell_ref(vertex) {
3890                let sheet_name = self.graph.sheet_name(cell.sheet_id);
3891                match self.read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1) {
3892                    Some(value) if !matches!(value, LiteralValue::Empty) => {
3893                        self.iterative_state_values.insert(vertex, value);
3894                    }
3895                    _ => {
3896                        self.iterative_state_values.remove(&vertex);
3897                    }
3898                }
3899            }
3900        }
3901        if !pending.is_empty() {
3902            self.graph.redirty_iterative_members(&pending);
3903        }
3904    }
3905
3906    /// Hash of every `EvalConfig` knob that can change the result of a
3907    /// retained SCC without any edit reaching the dependency graph. The
3908    /// function registry is tracked separately and precisely (see
3909    /// `retained_scc_function_epoch_seen`).
3910    fn retained_scc_config_fingerprint(&self) -> u64 {
3911        use std::hash::{Hash, Hasher};
3912        let mut hasher = rustc_hash::FxHasher::default();
3913        let config = &self.config;
3914        std::mem::discriminant(&config.cycle.detection).hash(&mut hasher);
3915        match config.cycle.policy {
3916            CyclePolicy::Error => 0u8.hash(&mut hasher),
3917            CyclePolicy::Iterate {
3918                max_iterations,
3919                max_change,
3920            } => {
3921                1u8.hash(&mut hasher);
3922                max_iterations.hash(&mut hasher);
3923                max_change.to_bits().hash(&mut hasher);
3924            }
3925        }
3926        config.date_system.hash(&mut hasher);
3927        config.workbook_seed.hash(&mut hasher);
3928        std::mem::discriminant(&config.volatile_level).hash(&mut hasher);
3929        // `DeterministicMode` carries a timestamp and a timezone spec; hash
3930        // its Debug rendering rather than growing its derive set for this.
3931        format!("{:?}", config.deterministic_mode).hash(&mut hasher);
3932        config.range_expansion_limit.hash(&mut hasher);
3933        config.max_open_ended_rows.hash(&mut hasher);
3934        config.max_open_ended_cols.hash(&mut hasher);
3935        hasher.finish()
3936    }
3937
3938    /// Request-begin bookkeeping for retained SCCs (#368): drop deleted
3939    /// vertices, invalidate everything when the config fingerprint moved
3940    /// (marking retained members dirty so their SCC tasks run in this
3941    /// request), and record how many retained SCCs are being reused, i.e.
3942    /// have no dirty member at request begin.
3943    fn reconcile_retained_sccs_at_request_begin(&mut self) {
3944        self.retained_scc_dirty_at_begin.clear();
3945        if self.retained_scc_members.is_empty() {
3946            return;
3947        }
3948        // Deleted vertices and members overwritten with a literal (the
3949        // vertex survives as a value cell) are no longer retained formulas.
3950        let graph = &self.graph;
3951        self.retained_scc_members
3952            .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3953        if self.retained_scc_config_fingerprint() != self.retained_scc_config_fingerprint {
3954            let members: Vec<VertexId> = self.retained_scc_members.keys().copied().collect();
3955            self.retained_scc_members.clear();
3956            self.graph.mark_dirty_many(&members);
3957            return;
3958        }
3959        let changes =
3960            crate::function_registry::semantic_changes_since(self.retained_scc_function_epoch_seen);
3961        let global_changed = changes.epoch != self.retained_scc_function_epoch_seen;
3962        let provider_revision = self.resolver.planning_semantic_revision();
3963        let provider_changed = provider_revision != self.retained_scc_provider_revision_seen;
3964        if global_changed || provider_changed {
3965            let changed: BTreeSet<(String, String)> = changes.keys.into_iter().collect();
3966            let affected: Vec<VertexId> = self
3967                .retained_scc_members
3968                .keys()
3969                .copied()
3970                .filter(|&vertex| {
3971                    let Some(ast) = self.graph.get_formula(vertex) else {
3972                        return true;
3973                    };
3974                    (global_changed
3975                        && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed)))
3976                        || (provider_changed && Self::ast_contains_function(&ast))
3977                })
3978                .collect();
3979            for vertex in &affected {
3980                self.retained_scc_members.remove(vertex);
3981            }
3982            if !affected.is_empty() {
3983                self.graph.mark_dirty_many(&affected);
3984            }
3985            self.retained_scc_function_epoch_seen = changes.epoch;
3986            self.retained_scc_provider_revision_seen = provider_revision;
3987        }
3988        let mut dirty_sccs: FxHashSet<u64> = FxHashSet::default();
3989        let mut all_sccs: FxHashSet<u64> = FxHashSet::default();
3990        for (&vertex, &scc) in &self.retained_scc_members {
3991            all_sccs.insert(scc);
3992            if self.graph.is_dirty(vertex) {
3993                dirty_sccs.insert(scc);
3994                self.retained_scc_dirty_at_begin.push((vertex, scc));
3995            }
3996        }
3997        let reused_members = self
3998            .retained_scc_members
3999            .values()
4000            .filter(|scc| !dirty_sccs.contains(scc))
4001            .count();
4002        let t = &mut self.last_cycle_telemetry;
4003        t.reused_sccs = all_sccs.len() - dirty_sccs.len();
4004        t.reused_scc_members = reused_members;
4005    }
4006
4007    pub fn virtual_dep_fallback_activations(&self) -> u64 {
4008        self.virtual_dep_fallback_activations
4009    }
4010
4011    #[cfg(test)]
4012    pub(crate) fn lookup_index_flights_built_for_test(&self) -> usize {
4013        self.lookup_index_cache
4014            .flights_built
4015            .load(std::sync::atomic::Ordering::Relaxed)
4016    }
4017
4018    pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
4019        self.lookup_index_cache.report()
4020    }
4021
4022    fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
4023        let start_row = view.start_row();
4024        let end_row = view.end_row();
4025        let start_col = view.start_col();
4026        let end_col = view.end_col();
4027        for row in start_row..=end_row {
4028            let Ok(row_u32) = u32::try_from(row) else {
4029                return true;
4030            };
4031            for col in start_col..=end_col {
4032                let Ok(col_u32) = u32::try_from(col) else {
4033                    return true;
4034                };
4035                let cell_ref = self
4036                    .graph
4037                    .make_cell_ref_internal(sheet_id, row_u32, col_u32);
4038                if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
4039                    && self.graph.is_volatile(vertex_id)
4040                {
4041                    return true;
4042                }
4043            }
4044        }
4045        false
4046    }
4047
4048    fn build_lookup_index_impl(
4049        &self,
4050        view: &RangeView<'_>,
4051        axis: LookupAxis,
4052    ) -> Option<Arc<LookupIndex>> {
4053        let (rows, cols) = view.dims();
4054        if rows == 0 || cols == 0 {
4055            self.lookup_index_cache.note_skipped_tiny();
4056            return None;
4057        }
4058        let len = match axis {
4059            LookupAxis::ColumnInView(col) => {
4060                if col >= cols {
4061                    self.lookup_index_cache.note_skipped_tiny();
4062                    return None;
4063                }
4064                rows
4065            }
4066            LookupAxis::RowInView(row) => {
4067                if row >= rows {
4068                    self.lookup_index_cache.note_skipped_tiny();
4069                    return None;
4070                }
4071                cols
4072            }
4073        };
4074        if len < 64 {
4075            self.lookup_index_cache.note_skipped_tiny();
4076            return None;
4077        }
4078
4079        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
4080        let key = LookupIndexKey {
4081            sheet_id,
4082            start_row: u32::try_from(view.start_row()).ok()?,
4083            start_col: u32::try_from(view.start_col()).ok()?,
4084            end_row: u32::try_from(view.end_row()).ok()?,
4085            end_col: u32::try_from(view.end_col()).ok()?,
4086            axis,
4087            snapshot_id: self.data_snapshot_id(),
4088        };
4089        if let Some(index) = self.lookup_index_cache.get(&key) {
4090            return Some(index);
4091        }
4092        if self
4093            .lookup_index_cache
4094            .would_exceed_cap(estimate_bytes(len, 0))
4095        {
4096            self.lookup_index_cache.note_skipped_cap();
4097            return None;
4098        }
4099        if !self.lookup_index_cache.should_build(key) {
4100            return None;
4101        }
4102        // Parallel members of a lookup family miss together: one builds.
4103        self.lookup_index_cache.single_flight(key, || {
4104            if let Some(index) = self.lookup_index_cache.recheck(&key) {
4105                return Some(index);
4106            }
4107            if self.lookup_index_cache.is_known_volatile(&key) {
4108                self.lookup_index_cache.note_skipped_volatile();
4109                return None;
4110            }
4111            if self.lookup_view_contains_volatile(view, sheet_id) {
4112                self.lookup_index_cache.note_volatile_key(key);
4113                self.lookup_index_cache.note_skipped_volatile();
4114                return None;
4115            }
4116            #[cfg(test)]
4117            self.lookup_index_cache
4118                .flights_built
4119                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4120            match LookupIndex::build(view, axis, self.config.date_system).ok()? {
4121                BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
4122                BuildOutcome::ErrorInLookupAxis => {
4123                    self.lookup_index_cache.note_skipped_error();
4124                    None
4125                }
4126                BuildOutcome::Degenerate => {
4127                    self.lookup_index_cache.note_skipped_tiny();
4128                    None
4129                }
4130            }
4131        })
4132    }
4133
4134    fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
4135        if !self.config.enable_virtual_dep_telemetry {
4136            self.last_virtual_dep_telemetry = VirtualDepTelemetry {
4137                fallback_mode_activations: self.virtual_dep_fallback_activations,
4138                ..VirtualDepTelemetry::default()
4139            };
4140        }
4141    }
4142
4143    fn source_cache_session(&self) -> SourceCacheSession {
4144        self.clear_source_cache();
4145        SourceCacheSession {
4146            cache: self.source_cache.clone(),
4147        }
4148    }
4149
4150    fn resolve_source_scalar_cached(
4151        &self,
4152        name: &str,
4153        version: Option<u64>,
4154    ) -> Result<LiteralValue, ExcelError> {
4155        let key = (name.to_string(), version);
4156        if let Ok(mut g) = self.source_cache.write() {
4157            if let Some(v) = g.scalars.get(&key) {
4158                return Ok(v.clone());
4159            }
4160
4161            let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
4162                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4163                    ExcelError::new(ExcelErrorKind::Ref)
4164                        .with_message(format!("Unresolved source scalar: {name}"))
4165                } else {
4166                    err
4167                }
4168            })?;
4169            g.scalars.insert(key, v.clone());
4170            Ok(v)
4171        } else {
4172            self.resolver.resolve_source_scalar(name).map_err(|err| {
4173                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4174                    ExcelError::new(ExcelErrorKind::Ref)
4175                        .with_message(format!("Unresolved source scalar: {name}"))
4176                } else {
4177                    err
4178                }
4179            })
4180        }
4181    }
4182
4183    fn resolve_source_table_cached(
4184        &self,
4185        name: &str,
4186        version: Option<u64>,
4187    ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
4188        let key = (name.to_string(), version);
4189        if let Ok(mut g) = self.source_cache.write() {
4190            if let Some(t) = g.tables.get(&key) {
4191                return Ok(t.clone());
4192            }
4193
4194            let t = self.resolver.resolve_source_table(name).map_err(|err| {
4195                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4196                    ExcelError::new(ExcelErrorKind::Ref)
4197                        .with_message(format!("Unresolved source table: {name}"))
4198                } else {
4199                    err
4200                }
4201            })?;
4202            let t: Arc<dyn crate::traits::Table> = Arc::from(t);
4203            g.tables.insert(key, t.clone());
4204            Ok(t)
4205        } else {
4206            self.resolver
4207                .resolve_source_table(name)
4208                .map_err(|err| {
4209                    if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4210                        ExcelError::new(ExcelErrorKind::Ref)
4211                            .with_message(format!("Unresolved source table: {name}"))
4212                    } else {
4213                        err
4214                    }
4215                })
4216                .map(Arc::from)
4217        }
4218    }
4219
4220    fn source_table_to_range_view(
4221        &self,
4222        table: &dyn crate::traits::Table,
4223        spec: &Option<formualizer_parse::parser::TableSpecifier>,
4224    ) -> Result<RangeView<'static>, ExcelError> {
4225        use formualizer_parse::parser::{SpecialItem, TableSpecifier};
4226
4227        let owned = match spec {
4228            Some(TableSpecifier::Column(c)) => {
4229                let c = c.trim();
4230                if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
4231                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4232                        "Complex structured references not yet supported".to_string(),
4233                    ));
4234                }
4235                table.get_column(c)?.materialise().into_owned()
4236            }
4237            Some(TableSpecifier::ColumnRange(start, end)) => {
4238                let cols = table.columns();
4239                let start = start.trim();
4240                let end = end.trim();
4241                let start_key = start.to_lowercase();
4242                let end_key = end.to_lowercase();
4243                let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
4244                let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
4245                if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
4246                    if si > ei {
4247                        std::mem::swap(&mut si, &mut ei);
4248                    }
4249                    let h = table.data_height();
4250                    let w = ei - si + 1;
4251                    let mut rows = vec![vec![LiteralValue::Empty; w]; h];
4252                    for (offset, ci) in (si..=ei).enumerate() {
4253                        let cname = &cols[ci];
4254                        let col_range = table.get_column(cname)?;
4255                        let (rh, _) = col_range.dimensions();
4256                        for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
4257                            row[offset] = col_range.get(r, 0)?;
4258                        }
4259                    }
4260                    rows
4261                } else {
4262                    return Err(ExcelError::new(ExcelErrorKind::Ref)
4263                        .with_message("Column range refers to unknown column(s)".to_string()));
4264                }
4265            }
4266            Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
4267            | Some(TableSpecifier::Headers) => table
4268                .headers_row()
4269                .map(|r| r.materialise().into_owned())
4270                .unwrap_or_default(),
4271            Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
4272            | Some(TableSpecifier::Totals) => table
4273                .totals_row()
4274                .map(|r| r.materialise().into_owned())
4275                .unwrap_or_default(),
4276            Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
4277                table
4278                    .data_body()
4279                    .map(|r| r.materialise().into_owned())
4280                    .unwrap_or_default()
4281            }
4282            Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
4283                let mut out: Vec<Vec<LiteralValue>> = Vec::new();
4284                if let Some(h) = table.headers_row() {
4285                    out.extend(h.iter_rows());
4286                }
4287                if let Some(body) = table.data_body() {
4288                    out.extend(body.iter_rows());
4289                }
4290                if let Some(tr) = table.totals_row() {
4291                    out.extend(tr.iter_rows());
4292                }
4293                out
4294            }
4295            Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
4296                return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4297                    "@ (This Row) requires table-aware context; not yet supported".to_string(),
4298                ));
4299            }
4300            Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
4301                return Err(ExcelError::new(ExcelErrorKind::NImpl)
4302                    .with_message("Complex structured references not yet supported".to_string()));
4303            }
4304            None => {
4305                return Err(ExcelError::new(ExcelErrorKind::NImpl)
4306                    .with_message("Table reference without specifier is unsupported".to_string()));
4307            }
4308        };
4309
4310        Ok(RangeView::from_owned_rows(owned, self.config.date_system))
4311    }
4312
4313    pub fn default_sheet_id(&self) -> SheetId {
4314        self.graph.default_sheet_id()
4315    }
4316
4317    pub fn default_sheet_name(&self) -> &str {
4318        self.graph.default_sheet_name()
4319    }
4320
4321    /// Update the workbook seed for deterministic RNGs in functions.
4322    pub fn set_workbook_seed(&mut self, seed: u64) {
4323        self.config.workbook_seed = seed;
4324    }
4325
4326    /// Set the volatile level policy (Always/OnRecalc/OnOpen)
4327    pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
4328        self.config.volatile_level = level;
4329    }
4330
4331    /// Set public temporal materialisation to native values or raw serials.
4332    pub fn set_temporal_egress(&mut self, policy: crate::engine::TemporalEgress) {
4333        self.config.temporal_egress = policy;
4334    }
4335
4336    pub fn temporal_egress(&self) -> crate::engine::TemporalEgress {
4337        self.config.temporal_egress
4338    }
4339
4340    /// Enable/disable deterministic evaluation mode (fixed clock + timezone).
4341    pub fn set_deterministic_mode(
4342        &mut self,
4343        mode: crate::engine::DeterministicMode,
4344    ) -> Result<(), ExcelError> {
4345        let clock = mode.build_clock()?;
4346        self.config.deterministic_mode = mode;
4347        self.clock = crate::timezone::SnapshotClock::new(clock);
4348        Ok(())
4349    }
4350
4351    /// Inject a custom [`ClockProvider`](crate::timezone::ClockProvider) for
4352    /// volatile date/time builtins (`NOW()`, `TODAY()`).
4353    ///
4354    /// The provider is the clock *source*; per spec §7.11 the engine samples
4355    /// it once at the start of every evaluation request and all reads within
4356    /// that recalc (including SCC iteration passes) observe the frozen
4357    /// sample.
4358    pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
4359        self.clock = crate::timezone::SnapshotClock::new(clock);
4360    }
4361
4362    fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
4363        self.config.deterministic_mode.validate()
4364    }
4365
4366    pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
4367        self.graph.sheet_id(name)
4368    }
4369
4370    pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
4371        self.add_sheet(name)
4372            .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
4373    }
4374
4375    pub fn sheet_name(&self, id: SheetId) -> &str {
4376        self.graph.sheet_name(id)
4377    }
4378
4379    fn arrow_sheet_position(&self, id: SheetId) -> Option<usize> {
4380        let index = self.arrow_sheet_index.get_or_init(|| {
4381            let mut index = Vec::new();
4382            for (position, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
4383                if let Some(id) = self.graph.sheet_id(&sheet.name) {
4384                    let needed = usize::from(id).saturating_add(1);
4385                    if index.len() < needed {
4386                        index.resize(needed, None);
4387                    }
4388                    if let Some(slot) = index.get_mut(usize::from(id)) {
4389                        *slot = Some(position);
4390                    }
4391                }
4392            }
4393            index
4394        });
4395        index.get(usize::from(id)).copied().flatten()
4396    }
4397
4398    /// Indexed Arrow storage lookup by stable registry identity.
4399    pub(crate) fn arrow_sheet_by_id(&self, id: SheetId) -> Option<&crate::arrow_store::ArrowSheet> {
4400        self.arrow_sheets.sheets.get(self.arrow_sheet_position(id)?)
4401    }
4402
4403    /// Mutable counterpart of [`Self::arrow_sheet_by_id`].
4404    pub(crate) fn arrow_sheet_by_id_mut(
4405        &mut self,
4406        id: SheetId,
4407    ) -> Option<&mut crate::arrow_store::ArrowSheet> {
4408        let position = self.arrow_sheet_position(id)?;
4409        self.arrow_sheets.sheets.get_mut(position)
4410    }
4411
4412    fn rebuild_arrow_sheet_index(&mut self) {
4413        self.arrow_sheet_index.take();
4414        self.resolved_arrow_sheets.clear();
4415        for (index, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
4416            if let Some(id) = self.graph.sheet_id(&sheet.name) {
4417                let len = usize::from(id).saturating_add(1);
4418                if self.resolved_arrow_sheets.len() < len {
4419                    self.resolved_arrow_sheets.resize(len, None);
4420                }
4421                if let Some(slot) = self.resolved_arrow_sheets.get_mut(usize::from(id)) {
4422                    *slot = Some(index);
4423                }
4424            }
4425        }
4426    }
4427
4428    pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
4429        let id = self.graph.add_sheet(name)?;
4430        self.ensure_arrow_sheet(name);
4431        self.mark_topology_edited();
4432        Ok(id)
4433    }
4434
4435    pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
4436        let source_id = self.graph.sheet_id(source).ok_or_else(|| {
4437            ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
4438        })?;
4439        if new_name.is_empty() || new_name.len() > 255 {
4440            return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
4441        }
4442        if self.graph.sheet_id(new_name).is_some() {
4443            return Err(ExcelError::new(ExcelErrorKind::Value)
4444                .with_message(format!("Sheet '{new_name}' already exists")));
4445        }
4446        let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
4447
4448        if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
4449            let mut copied_sheet = source_sheet;
4450            copied_sheet.name = Arc::<str>::from(new_name);
4451            self.arrow_sheets.sheets.push(copied_sheet);
4452            self.rebuild_arrow_sheet_index();
4453        } else {
4454            self.ensure_arrow_sheet(new_name);
4455        }
4456
4457        let duplicated_formulas = self
4458            .graph
4459            .formula_vertices()
4460            .into_iter()
4461            .filter(|vertex| {
4462                self.graph
4463                    .get_cell_ref(*vertex)
4464                    .is_some_and(|cell| cell.sheet_id == new_id)
4465            })
4466            .collect::<Vec<_>>();
4467        self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
4468        self.mark_topology_edited();
4469        Ok(new_id)
4470    }
4471
4472    fn ensure_arrow_sheet(&mut self, name: &str) {
4473        if self.arrow_sheets.sheet(name).is_some() {
4474            return;
4475        }
4476        self.arrow_sheets
4477            .sheets
4478            .push(crate::arrow_store::ArrowSheet {
4479                name: std::sync::Arc::<str>::from(name),
4480                date_system: self.config.date_system,
4481                columns: Vec::new(),
4482                nrows: 0,
4483                chunk_starts: Vec::new(),
4484                chunk_rows: 32 * 1024,
4485            });
4486        self.rebuild_arrow_sheet_index();
4487    }
4488
4489    pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
4490        let name = self.graph.sheet_name(sheet_id).to_string();
4491        self.purge_derived_formats_for_sheet(sheet_id);
4492        self.graph.remove_sheet(sheet_id)?;
4493        self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
4494        self.rebuild_arrow_sheet_index();
4495        // Sheet removal can change cross-sheet refs, names, and default-sheet
4496        // resolution. Until those domains have a complete exact dependency
4497        // proof, retain the documented graph-owned global invalidation.
4498        self.clear_all_computed_overlays();
4499        self.mark_all_formula_vertices_dirty();
4500        self.clear_staged_formulas_for_sheet(&name);
4501        if self.row_visibility.remove(&sheet_id).is_some() {
4502            self.invalidate_row_visibility_mask_cache();
4503        }
4504        self.record_structural_change(StructuralScope::RemovedSheet(sheet_id));
4505        self.mark_topology_edited();
4506        Ok(())
4507    }
4508
4509    /// Helper to synchronize the Arrow-backed storage layer.
4510    fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
4511        if let Some(asheet) = self
4512            .arrow_sheets
4513            .sheets
4514            .iter_mut()
4515            .find(|s| s.name.as_ref() == target_name)
4516        {
4517            asheet.name = std::sync::Arc::<str>::from(new_name);
4518            return true;
4519        }
4520        false
4521    }
4522
4523    pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4524        let old_name = self.graph.sheet_name(sheet_id).to_string();
4525
4526        // Speculative Storage Update
4527        // Update name in storage FIRST so the Evaluator can find it during Graph rescue.
4528        self.rename_sheet_in_arrow_store(&old_name, new_name);
4529
4530        // Graph Update (Metadata + Rescue Logic)
4531        match self.graph.rename_sheet(sheet_id, new_name) {
4532            Ok(_) => {
4533                self.rename_staged_formula_sheet(&old_name, new_name);
4534                // Success! Invalidate cache for the moved sheet
4535                let sheet_vertices: Vec<VertexId> = self
4536                    .graph
4537                    .grid_vertices_in_sheet(sheet_id)
4538                    .map(|(id, _)| id)
4539                    .collect();
4540                for v_id in sheet_vertices {
4541                    self.graph.mark_vertex_dirty(v_id);
4542                }
4543                // Sheet rename preserves SheetId and therefore formula dependencies.
4544                self.mark_topology_edited();
4545                Ok(())
4546            }
4547            Err(e) => {
4548                // ROLLBACK: Revert storage if graph rejected the name
4549                self.rename_sheet_in_arrow_store(new_name, &old_name);
4550                Err(e)
4551            }
4552        }
4553    }
4554
4555    pub fn named_ranges_iter(
4556        &self,
4557    ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4558        self.graph.named_ranges_iter()
4559    }
4560
4561    pub fn sheet_named_ranges_iter(
4562        &self,
4563    ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4564        self.graph.sheet_named_ranges_iter()
4565    }
4566
4567    pub fn resolve_name_entry(
4568        &self,
4569        name: &str,
4570        current_sheet: SheetId,
4571    ) -> Option<&crate::engine::named_range::NamedRange> {
4572        self.graph.resolve_name_entry(name, current_sheet)
4573    }
4574
4575    /// The [`NameScope`] an optional scope-sheet argument denotes.
4576    ///
4577    /// `None` means **workbook scope**, not "the default sheet": a caller that
4578    /// supplies no sheet context is asking about workbook-scoped names only.
4579    /// An unknown sheet name is a malformed query and errors rather than
4580    /// silently degrading to another sheet's scope (issue #110).
4581    ///
4582    /// This is the one owned derivation from `Option<&str>` to a name scope;
4583    /// every scope-taking entry point routes through it.
4584    pub(crate) fn name_query_scope(
4585        &self,
4586        scope_sheet: Option<&str>,
4587    ) -> Result<NameScope, ExcelError> {
4588        match scope_sheet {
4589            None => Ok(NameScope::Workbook),
4590            Some(sheet) => self
4591                .graph
4592                .sheet_id(sheet)
4593                .map(NameScope::Sheet)
4594                .ok_or_else(|| {
4595                    ExcelError::new(ExcelErrorKind::Ref)
4596                        .with_message(format!("name scope sheet not found: {sheet}"))
4597                }),
4598        }
4599    }
4600
4601    /// Whether `name` resolves in the scope denoted by `scope_sheet`.
4602    ///
4603    /// `scope_sheet == None` asks about workbook scope only; a name scoped to a
4604    /// single sheet (including the default sheet) does not answer it. An unknown
4605    /// sheet name resolves nothing.
4606    /// Resolve a [`SharedSheetLocator`](crate::reference::SharedSheetLocator)
4607    /// against an explicit context sheet.
4608    ///
4609    /// Thin forwarder to
4610    /// [`SheetRegistry::resolve_locator`](crate::engine::sheet_registry::SheetRegistry::resolve_locator),
4611    /// the single owned derivation. `Current` resolves to `context_sheet`, never
4612    /// to the workbook's default sheet.
4613    fn resolve_sheet_locator(
4614        &self,
4615        locator: &crate::reference::SharedSheetLocator<'_>,
4616        context_sheet: SheetId,
4617    ) -> Result<SheetId, ExcelError> {
4618        self.graph
4619            .sheet_reg()
4620            .resolve_locator(locator, context_sheet)
4621    }
4622
4623    pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4624        let Ok(scope) = self.name_query_scope(scope_sheet) else {
4625            return false;
4626        };
4627        self.graph
4628            .resolve_name_entry_in_scope(name, scope)
4629            .is_some()
4630    }
4631
4632    /// The current value of `name` in the scope denoted by `scope_sheet`.
4633    ///
4634    /// Scoping follows [`Self::has_name`]: `None` is workbook scope only.
4635    pub fn resolved_name_value(
4636        &self,
4637        name: &str,
4638        scope_sheet: Option<&str>,
4639    ) -> Option<LiteralValue> {
4640        let scope = self.name_query_scope(scope_sheet).ok()?;
4641        let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4642        self.graph.get_value(entry.vertex)
4643    }
4644
4645    pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4646        let entry = self.graph.resolve_table_entry(name)?;
4647        Some(TableMetadata {
4648            name: entry.name.clone(),
4649            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4650            start_row: entry.range.start.coord.row() + 1,
4651            start_col: entry.range.start.coord.col() + 1,
4652            end_row: entry.range.end.coord.row() + 1,
4653            end_col: entry.range.end.coord.col() + 1,
4654            header_row: entry.header_row,
4655            headers: entry.headers.clone(),
4656            totals_row: entry.totals_row,
4657        })
4658    }
4659
4660    /// Metadata for every defined table, ordered by name.
4661    pub fn tables(&self) -> Vec<TableMetadata> {
4662        self.graph
4663            .table_names()
4664            .into_iter()
4665            .filter_map(|name| self.table_metadata(&name))
4666            .collect()
4667    }
4668
4669    pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4670        let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4671
4672        for (name, named) in self.graph.named_ranges_iter() {
4673            out.push(crate::engine::named_range::NamedRangeSnapshot {
4674                name: name.clone(),
4675                scope: NameScope::Workbook,
4676                definition: named.definition.clone(),
4677            });
4678        }
4679
4680        for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4681            out.push(crate::engine::named_range::NamedRangeSnapshot {
4682                name: name.clone(),
4683                scope: NameScope::Sheet(*sheet_id),
4684                definition: named.definition.clone(),
4685            });
4686        }
4687
4688        out.sort_by(|a, b| {
4689            let a_scope = match a.scope {
4690                NameScope::Workbook => (0u8, 0u32),
4691                NameScope::Sheet(id) => (1u8, u32::from(id)),
4692            };
4693            let b_scope = match b.scope {
4694                NameScope::Workbook => (0u8, 0u32),
4695                NameScope::Sheet(id) => (1u8, u32::from(id)),
4696            };
4697            a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4698        });
4699
4700        out
4701    }
4702
4703    pub fn named_ranges_snapshot_for_sheet(
4704        &self,
4705        sheet_id: SheetId,
4706    ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4707        self.named_ranges_snapshot()
4708            .into_iter()
4709            .filter(|entry| match entry.scope {
4710                NameScope::Workbook => true,
4711                NameScope::Sheet(id) => id == sheet_id,
4712            })
4713            .collect()
4714    }
4715
4716    pub fn define_name(
4717        &mut self,
4718        name: &str,
4719        definition: NamedDefinition,
4720        scope: NameScope,
4721    ) -> Result<(), ExcelError> {
4722        self.graph.validate_define_name(name, scope)?;
4723        self.graph.define_name(name, definition, scope)?;
4724        self.record_structural_change(StructuralScope::AllSheets);
4725
4726        self.mark_topology_edited();
4727
4728        Ok(())
4729    }
4730
4731    pub fn update_name(
4732        &mut self,
4733        name: &str,
4734        definition: NamedDefinition,
4735        scope: NameScope,
4736    ) -> Result<(), ExcelError> {
4737        self.graph.validate_existing_name(name, scope)?;
4738        self.graph.update_name(name, definition, scope)?;
4739        self.record_structural_change(StructuralScope::AllSheets);
4740
4741        self.mark_topology_edited();
4742
4743        Ok(())
4744    }
4745
4746    pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4747        self.graph.validate_existing_name(name, scope)?;
4748        self.graph.delete_name(name, scope)?;
4749        self.record_structural_change(StructuralScope::AllSheets);
4750
4751        self.mark_topology_edited();
4752
4753        Ok(())
4754    }
4755
4756    pub fn define_table(
4757        &mut self,
4758        name: &str,
4759        range: crate::reference::RangeRef,
4760        header_row: bool,
4761        headers: Vec<String>,
4762        totals_row: bool,
4763    ) -> Result<(), ExcelError> {
4764        self.graph
4765            .define_table(name, range, header_row, headers, totals_row)?;
4766        self.record_structural_change(StructuralScope::AllSheets);
4767        self.mark_topology_edited();
4768        Ok(())
4769    }
4770
4771    pub fn define_source_scalar(
4772        &mut self,
4773        name: &str,
4774        version: Option<u64>,
4775    ) -> Result<(), ExcelError> {
4776        self.graph.define_source_scalar(name, version)?;
4777        self.record_structural_change(StructuralScope::OpaqueGlobal);
4778        self.mark_topology_edited();
4779        Ok(())
4780    }
4781
4782    pub fn define_source_table(
4783        &mut self,
4784        name: &str,
4785        version: Option<u64>,
4786    ) -> Result<(), ExcelError> {
4787        self.graph.define_source_table(name, version)?;
4788        self.record_structural_change(StructuralScope::OpaqueGlobal);
4789        self.mark_topology_edited();
4790        Ok(())
4791    }
4792
4793    pub fn set_source_scalar_version(
4794        &mut self,
4795        name: &str,
4796        version: Option<u64>,
4797    ) -> Result<(), ExcelError> {
4798        self.graph.set_source_scalar_version(name, version)?;
4799        Ok(())
4800    }
4801
4802    pub fn set_source_table_version(
4803        &mut self,
4804        name: &str,
4805        version: Option<u64>,
4806    ) -> Result<(), ExcelError> {
4807        self.graph.set_source_table_version(name, version)?;
4808        Ok(())
4809    }
4810
4811    pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4812        self.graph.invalidate_source(name)?;
4813        Ok(())
4814    }
4815
4816    pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4817        self.graph.get_value(vertex)
4818    }
4819
4820    pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4821        self.graph.get_cell_value(sheet, row, col)
4822    }
4823
4824    pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4825        self.graph.get_vertex_for_cell(cell)
4826    }
4827
4828    pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4829        self.graph.get_evaluation_vertices()
4830    }
4831
4832    /// Return read-only baseline counters for dispatch benchmarking.
4833    pub fn baseline_stats(&self) -> EngineBaselineStats {
4834        let graph = self.graph.baseline_stats();
4835        EngineBaselineStats {
4836            graph_vertex_count: graph.graph_vertex_count,
4837            graph_formula_vertex_count: graph.graph_formula_vertex_count,
4838            graph_edge_count: graph.graph_edge_count,
4839            dirty_vertex_count: graph.dirty_vertex_count,
4840            evaluation_vertex_count: graph.evaluation_vertex_count,
4841            formula_ast_root_count: graph.formula_ast_root_count,
4842            formula_ast_node_count: graph.formula_ast_node_count,
4843            staged_formula_count: self.staged_formula_count(),
4844            formula_plane_active_span_count: 0,
4845            formula_plane_producer_result_entries: 0,
4846            formula_plane_consumer_read_entries: 0,
4847            formula_plane_mixed_topology_cache_builds: 0,
4848            formula_plane_mixed_topology_cache_hits: 0,
4849            formula_plane_mixed_topology_cache_overflows: 0,
4850            formula_plane_dirty_pending_events: 0,
4851            formula_plane_dirty_region_events_recorded: 0,
4852            formula_plane_dirty_span_region_events_recorded: 0,
4853            formula_plane_dirty_whole_span_seeds_recorded: 0,
4854            formula_plane_dirty_global_invalidations: 0,
4855            formula_plane_structural_span_candidates: 0,
4856            formula_plane_cycle_member_span_demotions: 0,
4857            formula_plane_array_result_span_demotions: 0,
4858            retained_scc_members: self.retained_scc_members.len(),
4859        }
4860    }
4861
4862    /// Mutation revision captured by read-only engine reports.
4863    pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4864        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4865    }
4866
4867    #[cfg(test)]
4868    pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4869        self.used_axis_bounds_cache
4870            .read()
4871            .ok()
4872            .and_then(|guard| {
4873                guard.as_ref().map(|cache| {
4874                    (
4875                        cache.row_hits.load(Ordering::Relaxed),
4876                        cache.row_misses.load(Ordering::Relaxed),
4877                        cache.col_hits.load(Ordering::Relaxed),
4878                        cache.col_misses.load(Ordering::Relaxed),
4879                    )
4880                })
4881            })
4882            .unwrap_or((0, 0, 0, 0))
4883    }
4884
4885    pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4886        self.graph.set_first_load_assume_new(enabled);
4887    }
4888
4889    pub fn first_load_assume_new(&self) -> bool {
4890        self.graph.first_load_assume_new()
4891    }
4892
4893    pub fn reset_ensure_touched(&mut self) {
4894        self.graph.reset_ensure_touched();
4895    }
4896
4897    pub fn finalize_sheet_index(&mut self, sheet: &str) {
4898        self.graph.finalize_sheet_index(sheet);
4899    }
4900
4901    /// Rebuild the requested sheet indexes in a single coordinate-map pass.
4902    #[doc(hidden)]
4903    pub fn finalize_all_sheet_indexes<'a>(&mut self, sheets: impl IntoIterator<Item = &'a str>) {
4904        self.graph.finalize_all_sheet_indexes(sheets);
4905    }
4906
4907    /// Execute a named Engine action.
4908    ///
4909    /// Ticket 614 introduces this as the stable Engine-level transaction surface.
4910    /// For now actions are commit-only: they do not create changelog boundaries and they do not
4911    /// provide rollback/atomicity.
4912    ///
4913    /// Nested actions are deterministically handled by *disallowing* nesting: calling
4914    /// `Engine::action` while another action is active returns `EditorError::TransactionFailed`.
4915    pub fn action<T>(
4916        &mut self,
4917        name: impl AsRef<str>,
4918        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4919    ) -> Result<T, crate::engine::EditorError> {
4920        if self.action_depth != 0 {
4921            return Err(crate::engine::EditorError::TransactionFailed {
4922                reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4923                    .to_string(),
4924            });
4925        }
4926
4927        self.action_depth = 1;
4928        let engine_ptr: *mut Engine<R> = self;
4929        let _guard = ActionDepthGuard {
4930            engine: engine_ptr,
4931            _marker: std::marker::PhantomData,
4932        };
4933
4934        let mut tx = EngineAction {
4935            engine: self,
4936            name: name.as_ref().to_string(),
4937            capture: None,
4938            arrow_undo: None,
4939            atomic_policy: false,
4940        };
4941        f(&mut tx)
4942    }
4943
4944    /// Execute a named Engine action with atomic commit/rollback semantics.
4945    ///
4946    /// This variant does not require a `ChangeLog` and uses an internal journal for rollback.
4947    pub fn action_atomic<T>(
4948        &mut self,
4949        name: impl Into<String>,
4950        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4951    ) -> Result<T, crate::engine::EditorError> {
4952        let (v, _j) = self.action_atomic_journal(name, f)?;
4953        Ok(v)
4954    }
4955
4956    /// Like `action_atomic`, but returns the committed journal entry for undo/redo storage.
4957    pub fn action_atomic_journal<T>(
4958        &mut self,
4959        name: impl Into<String>,
4960        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4961    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4962        if self.action_depth != 0 {
4963            return Err(crate::engine::EditorError::TransactionFailed {
4964                reason: "Nested Engine::action calls are not supported (deterministic rule)"
4965                    .to_string(),
4966            });
4967        }
4968
4969        self.action_depth = 1;
4970        let engine_ptr: *mut Engine<R> = self;
4971        let _guard = ActionDepthGuard {
4972            engine: engine_ptr,
4973            _marker: std::marker::PhantomData,
4974        };
4975
4976        let name_str = name.into();
4977        let mut capture = MutationCapture::new(Default::default());
4978        let start_len = capture.len();
4979        self.action_atomic_impl(&mut capture, start_len, true, name_str, f)
4980    }
4981
4982    fn action_atomic_impl<T>(
4983        &mut self,
4984        capture: &mut MutationCapture,
4985        start_len: usize,
4986        expand_runs: bool,
4987        name: String,
4988        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4989    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4990        let invalidation_baseline = self.invalidation_baseline();
4991        let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4992        let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4993
4994        let capture_ptr: *mut MutationCapture = capture;
4995        let mut tx = EngineAction {
4996            engine: self,
4997            name: name.clone(),
4998            capture: Some(capture_ptr),
4999            arrow_undo: Some(arrow_ptr),
5000            atomic_policy: true,
5001        };
5002
5003        let res = f(&mut tx);
5004
5005        // Capture graph structural delta for this action. The journal is a
5006        // public value: run records (Program 2) are expanded into it, except
5007        // for a caller that only publishes the capture to its change log
5008        // (`expand_runs` false), where the journal (plain events) drives
5009        // invalidation and records count as topology changes; a rollback
5010        // still replays the expanded events.
5011        let capture_ref = unsafe { &*capture_ptr };
5012        let has_runs = capture_ref.lazy_len() > 0;
5013        let graph_events: Vec<crate::engine::ChangeEvent> = if expand_runs || res.is_err() {
5014            capture_ref.expanded_events_from(start_len, 0)
5015        } else {
5016            capture_ref.events()[start_len..].to_vec()
5017        };
5018        let graph_batch = crate::engine::GraphUndoBatch {
5019            events: graph_events,
5020        };
5021        let affected_cells = arrow_undo.ops.len();
5022        let journal = crate::engine::ActionJournal {
5023            name,
5024            graph: graph_batch,
5025            arrow: arrow_undo,
5026            affected_cells,
5027        };
5028
5029        match res {
5030            Ok(v) => {
5031                if !journal.graph.is_empty() || !journal.arrow.is_empty() || has_runs {
5032                    for event in &journal.graph.events {
5033                        self.record_change_for_event(event);
5034                    }
5035                    let mut impact = Self::classify_change_events(
5036                        &journal.graph.events,
5037                        LoggedEditDirection::Original,
5038                    )
5039                    .max(Self::classify_arrow_undo(&journal.arrow));
5040                    if has_runs {
5041                        impact = impact.max(LoggedEditImpact::Topology);
5042                    }
5043                    self.apply_logged_edit_impact(impact, invalidation_baseline);
5044                }
5045                Ok((v, journal))
5046            }
5047            Err(e) => {
5048                if let Err(rb) = self.rollback_from_action_journal(&journal, invalidation_baseline)
5049                {
5050                    return Err(crate::engine::EditorError::TransactionFailed {
5051                        reason: format!(
5052                            "Engine::action_atomic rollback failed after error '{e}': {rb}"
5053                        ),
5054                    });
5055                }
5056                if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5057                    for event in &journal.graph.events {
5058                        self.record_change_for_event(event);
5059                    }
5060                }
5061                Err(e)
5062            }
5063        }
5064    }
5065
5066    /// Execute a named Engine action, logging graph changes into the provided ChangeLog.
5067    ///
5068    /// Ticket 615: this variant provides atomicity. If the action returns an error, it rolls back:
5069    /// - Dependency graph structural edits (via inverse ChangeEvents)
5070    /// - Arrow-truth overlay writes mirrored from ChangeEvents
5071    /// - ChangeLog entries (published only after a successful commit)
5072    pub fn action_with_logger<T>(
5073        &mut self,
5074        log: &mut crate::engine::ChangeLog,
5075        name: impl AsRef<str>,
5076        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5077    ) -> Result<T, crate::engine::EditorError> {
5078        if self.action_depth != 0 {
5079            return Err(crate::engine::EditorError::TransactionFailed {
5080                reason: "Nested Engine::action calls are not supported (deterministic rule)"
5081                    .to_string(),
5082            });
5083        }
5084
5085        self.action_depth = 1;
5086        let engine_ptr: *mut Engine<R> = self;
5087        let _guard = ActionDepthGuard {
5088            engine: engine_ptr,
5089            _marker: std::marker::PhantomData,
5090        };
5091
5092        let name_str = name.as_ref().to_string();
5093        let mut capture = MutationCapture::new(log.current_meta());
5094        let start_len = capture.len();
5095        capture.begin_compound(name_str.clone());
5096
5097        // Mutation correctness uses the complete private capture. The provided ChangeLog remains
5098        // an observability sink and is not touched until the action outcome is known.
5099        let res = self.action_atomic_impl(&mut capture, start_len, false, name_str, f);
5100        capture.close_compounds();
5101
5102        match res {
5103            Ok((v, _journal)) => {
5104                log.publish_capture(capture);
5105                Ok(v)
5106            }
5107            Err(e) => {
5108                // Preserve sequence/group gaps without retaining failed events or evicting history.
5109                log.discard_capture(capture);
5110                Err(e)
5111            }
5112        }
5113    }
5114
5115    fn rollback_from_action_journal(
5116        &mut self,
5117        journal: &crate::engine::ActionJournal,
5118        invalidation_baseline: InvalidationBaseline,
5119    ) -> Result<(), crate::engine::EditorError> {
5120        // Invalidate first so a partial inverse failure cannot leave a changed
5121        // graph behind an apparently current schedule or lookup cache.
5122        self.invalidate_for_action_journal(
5123            journal,
5124            LoggedEditDirection::InverseReplay,
5125            invalidation_baseline,
5126        );
5127        // 1) Roll back the dependency graph structure.
5128        journal.graph.undo(&mut self.graph)?;
5129        // 2) Roll back engine row-visibility sidecar events.
5130        self.apply_inverse_row_visibility_events(&journal.graph.events);
5131        // 3) Roll back Arrow-truth overlays.
5132        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
5133        Ok(())
5134    }
5135
5136    fn rollback_from_change_events(
5137        &mut self,
5138        events: &[crate::engine::ChangeEvent],
5139        invalidation_baseline: InvalidationBaseline,
5140    ) -> Result<(), crate::engine::EditorError> {
5141        use crate::engine::ChangeEvent;
5142
5143        // Fail closed before applying inverses because replay can return after
5144        // only part of the batch has been restored.
5145        self.invalidate_for_change_events(
5146            events,
5147            LoggedEditDirection::InverseReplay,
5148            invalidation_baseline,
5149        );
5150
5151        // 1) Roll back the dependency graph.
5152        self.graph
5153            .authority_set_replay(crate::engine::authority::history::Replay::Undo);
5154        let rolled_back = (|| {
5155            let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
5156            let mut compound_stack: Vec<usize> = Vec::new();
5157            for (i, ev) in events.iter().enumerate().rev() {
5158                match ev {
5159                    ChangeEvent::CompoundEnd { depth } => {
5160                        compound_stack.push(*depth);
5161                        if let Some(description) =
5162                            crate::engine::graph::editor::change_log::compound_start_description(
5163                                i,
5164                                |j| &events[j],
5165                            )
5166                        {
5167                            editor.inverse_compound_end(description);
5168                        }
5169                    }
5170                    ChangeEvent::CompoundStart { depth, .. } => {
5171                        if compound_stack.last() == Some(depth) {
5172                            compound_stack.pop();
5173                        }
5174                        editor.apply_inverse(ev.clone())?;
5175                    }
5176                    ChangeEvent::SetRowVisibility { .. } => {
5177                        // Engine-side metadata handled after dropping graph editor borrow.
5178                    }
5179                    _ => {
5180                        editor.apply_inverse(ev.clone())?;
5181                    }
5182                }
5183            }
5184            Ok::<_, crate::engine::EditorError>(())
5185        })();
5186        self.graph
5187            .authority_set_replay(crate::engine::authority::history::Replay::Forward);
5188        rolled_back?;
5189
5190        // 2) Roll back engine row-visibility metadata.
5191        for ev in events.iter().rev() {
5192            self.apply_inverse_row_visibility_event(ev);
5193        }
5194
5195        // 3) Roll back Arrow-truth overlays mirrored from those ChangeEvents.
5196        for ev in events.iter().rev() {
5197            self.mirror_inverse_change_to_arrow(ev);
5198        }
5199
5200        Ok(())
5201    }
5202
5203    fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
5204        let sheet_id = self.graph.sheet_id(sheet)?;
5205        let coord = Coord::from_excel(row, col, true, true);
5206        let cell = CellRef::new(sheet_id, coord);
5207        let vid = self.graph.get_vertex_for_cell(&cell)?;
5208        self.graph.get_formula(vid)
5209    }
5210
5211    pub fn define_name_with_logger(
5212        &mut self,
5213        log: &mut crate::engine::ChangeLog,
5214        name: &str,
5215        definition: NamedDefinition,
5216        scope: NameScope,
5217    ) -> Result<(), crate::engine::EditorError> {
5218        self.graph
5219            .validate_define_name(name, scope)
5220            .map_err(crate::engine::EditorError::Excel)?;
5221
5222        {
5223            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5224            editor.define_name(name, definition, scope)?;
5225        }
5226        self.record_structural_change(StructuralScope::AllSheets);
5227
5228        self.mark_topology_edited();
5229
5230        Ok(())
5231    }
5232
5233    pub fn update_name_with_logger(
5234        &mut self,
5235        log: &mut crate::engine::ChangeLog,
5236        name: &str,
5237        definition: NamedDefinition,
5238        scope: NameScope,
5239    ) -> Result<(), crate::engine::EditorError> {
5240        self.graph
5241            .validate_existing_name(name, scope)
5242            .map_err(crate::engine::EditorError::Excel)?;
5243        {
5244            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5245            editor.update_name(name, definition, scope)?;
5246        }
5247        self.record_structural_change(StructuralScope::AllSheets);
5248
5249        self.mark_topology_edited();
5250
5251        Ok(())
5252    }
5253
5254    pub fn delete_name_with_logger(
5255        &mut self,
5256        log: &mut crate::engine::ChangeLog,
5257        name: &str,
5258        scope: NameScope,
5259    ) -> Result<(), crate::engine::EditorError> {
5260        self.graph
5261            .validate_existing_name(name, scope)
5262            .map_err(crate::engine::EditorError::Excel)?;
5263        {
5264            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5265            editor.delete_name(name, scope)?;
5266        }
5267        self.record_structural_change(StructuralScope::AllSheets);
5268
5269        self.mark_topology_edited();
5270
5271        Ok(())
5272    }
5273
5274    pub fn edit_with_logger<T>(
5275        &mut self,
5276        log: &mut crate::engine::ChangeLog,
5277        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5278    ) -> Result<T, crate::engine::EditorError> {
5279        let mut capture = MutationCapture::new(log.current_meta());
5280        let result = self.edit_with_capture(&mut capture, f);
5281        capture.close_compounds();
5282        match result {
5283            Ok(value) => {
5284                log.publish_capture(capture);
5285                Ok(value)
5286            }
5287            Err(error) => {
5288                log.discard_capture(capture);
5289                Err(error)
5290            }
5291        }
5292    }
5293
5294    fn edit_with_capture<T>(
5295        &mut self,
5296        capture: &mut MutationCapture,
5297        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5298    ) -> Result<T, crate::engine::EditorError> {
5299        let invalidation_baseline = self.invalidation_baseline();
5300        let start_len = capture.len();
5301        let lazy_start = capture.lazy_len();
5302
5303        // Provide a spill snapshot reader so VertexEditor can snapshot Arrow-truth spill values
5304        // (graph value cache is intentionally empty in canonical mode).
5305        struct ArrowSpillReader<'a> {
5306            sheets: &'a crate::arrow_store::SheetStore,
5307        }
5308        impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
5309            fn read_cell_value(
5310                &self,
5311                sheet: &str,
5312                row: u32,
5313                col: u32,
5314            ) -> Option<formualizer_common::LiteralValue> {
5315                use formualizer_common::LiteralValue;
5316                let asheet = self.sheets.sheet(sheet)?;
5317                let r0 = row.saturating_sub(1) as usize;
5318                let c0 = col.saturating_sub(1) as usize;
5319                let v = asheet.get_cell_value(r0, c0);
5320                if matches!(v, LiteralValue::Empty) {
5321                    None
5322                } else {
5323                    Some(v)
5324                }
5325            }
5326        }
5327
5328        let ret = {
5329            let spill_reader = ArrowSpillReader {
5330                sheets: &self.arrow_sheets,
5331            };
5332            let mut editor = crate::engine::VertexEditor::with_capture_and_spill_reader(
5333                &mut self.graph,
5334                capture,
5335                &spill_reader,
5336            );
5337            f(&mut editor)
5338        };
5339
5340        // Plain events only: run records (Program 2) stand for
5341        // `FormulaAdjusted` events, which have no forward effect here but
5342        // topology invalidation.
5343        let new_events = capture.events()[start_len..].to_vec();
5344        let new_runs = capture.lazy_len() > lazy_start;
5345        if new_events.iter().any(|event| {
5346            matches!(
5347                event,
5348                ChangeEvent::DefineName { .. }
5349                    | ChangeEvent::UpdateName { .. }
5350                    | ChangeEvent::DeleteName { .. }
5351            )
5352        }) {
5353            let all = capture.expanded_events_from(start_len, lazy_start);
5354            self.rollback_from_change_events(&all, invalidation_baseline)?;
5355            return Err(crate::engine::EditorError::TransactionUnsupported {
5356                reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
5357            });
5358        }
5359
5360        // Mirror value-impacting graph events to Arrow for forward edits.
5361        // This keeps Arrow overlays (delta + computed) consistent when edits clear/commit spills.
5362        self.clear_logged_cell_format_states(&new_events);
5363        for ev in &new_events {
5364            self.mirror_forward_change_to_arrow(ev);
5365        }
5366        for ev in &new_events {
5367            self.record_change_for_event(ev);
5368        }
5369
5370        // Atomic EngineAction calls publish one invalidation for their complete
5371        // journal at commit/rollback. Direct logged edits publish here.
5372        if self.action_depth == 0 {
5373            let mut impact =
5374                Self::classify_change_events(&new_events, LoggedEditDirection::Original);
5375            if new_runs {
5376                impact = impact.max(LoggedEditImpact::Topology);
5377            }
5378            self.apply_logged_edit_impact(impact, invalidation_baseline);
5379        }
5380
5381        Ok(ret)
5382    }
5383
5384    pub(crate) fn preflight_replay_admission(
5385        &mut self,
5386        events: &[ChangeEvent],
5387        forward: bool,
5388    ) -> Result<(), crate::engine::EditorError> {
5389        if !self.graph_admission_enabled() {
5390            return Ok(());
5391        }
5392        let mut vertex_delta = 0i128;
5393        let mut edge_delta = 0i128;
5394        let mut added_vertices = 0usize;
5395        let mut added_edges = 0usize;
5396        let mut formula_cells = BTreeSet::new();
5397        for event in events {
5398            match event {
5399                ChangeEvent::AddVertex {
5400                    formula,
5401                    coord,
5402                    sheet_id,
5403                    ..
5404                } => {
5405                    let delta = if forward { 1 } else { -1 };
5406                    vertex_delta += delta;
5407                    if forward {
5408                        added_vertices = added_vertices.saturating_add(1);
5409                        if formula.is_some() {
5410                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5411                        }
5412                    }
5413                }
5414                ChangeEvent::RemoveVertex {
5415                    old_formula,
5416                    coord,
5417                    sheet_id,
5418                    ..
5419                } => {
5420                    let delta = if forward { -1 } else { 1 };
5421                    vertex_delta += delta;
5422                    if !forward {
5423                        added_vertices = added_vertices.saturating_add(1);
5424                        if old_formula.is_some()
5425                            && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
5426                        {
5427                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5428                        }
5429                    }
5430                }
5431                ChangeEvent::EdgeAdded { .. } => {
5432                    let delta = if forward { 1 } else { -1 };
5433                    edge_delta += delta;
5434                    if forward {
5435                        added_edges = added_edges.saturating_add(1);
5436                    }
5437                }
5438                ChangeEvent::EdgeRemoved { .. } => {
5439                    let delta = if forward { -1 } else { 1 };
5440                    edge_delta += delta;
5441                    if !forward {
5442                        added_edges = added_edges.saturating_add(1);
5443                    }
5444                }
5445                ChangeEvent::SetFormula {
5446                    addr, old_formula, ..
5447                } if (forward || old_formula.is_some()) => {
5448                    formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
5449                }
5450                _ => {}
5451            }
5452        }
5453        let stats = self.graph.baseline_stats();
5454        let final_vertices = i128::try_from(stats.graph_vertex_count)
5455            .ok()
5456            .and_then(|count| count.checked_add(vertex_delta))
5457            .and_then(|count| usize::try_from(count).ok())
5458            .ok_or_else(|| {
5459                crate::engine::EditorError::Excel(
5460                    ExcelError::new(ExcelErrorKind::NImpl)
5461                        .with_message("replay vertex count overflow"),
5462                )
5463            })?;
5464        let final_edges = i128::try_from(stats.graph_edge_count)
5465            .ok()
5466            .and_then(|count| count.checked_add(edge_delta))
5467            .and_then(|count| usize::try_from(count).ok())
5468            .ok_or_else(|| {
5469                crate::engine::EditorError::Excel(
5470                    ExcelError::new(ExcelErrorKind::NImpl)
5471                        .with_message("replay edge count overflow"),
5472                )
5473            })?;
5474        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5475            final_vertices,
5476            final_edges,
5477            materialization_cells: formula_cells.len() as u64,
5478            added_vertices,
5479            added_edges,
5480        })
5481        .map_err(crate::engine::EditorError::Excel)
5482    }
5483
5484    /// Undo the last group still retained by the provided audit log.
5485    ///
5486    /// Disabled, zero-cap, and evicted history is unavailable on this index-based path. Use an
5487    /// explicit `ActionJournal` with `undo_action` when undo must be independent of audit retention.
5488    pub fn undo_logged(
5489        &mut self,
5490        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5491        log: &mut crate::engine::ChangeLog,
5492    ) -> Result<(), crate::engine::EditorError> {
5493        let pending_events = log
5494            .last_group_indices()
5495            .into_iter()
5496            .map(|index| log.events()[index].clone())
5497            .collect::<Vec<_>>();
5498        self.preflight_replay_admission(&pending_events, false)?;
5499        let invalidation_baseline = self.invalidation_baseline();
5500        // UndoEngine can fail after partially applying the batch, so publish
5501        // invalidation before replay rather than only on the success path.
5502        self.invalidate_for_change_events(
5503            &pending_events,
5504            LoggedEditDirection::InverseReplay,
5505            invalidation_baseline,
5506        );
5507        let batch = undo.undo(&mut self.graph, log)?;
5508        for item in batch.iter().rev() {
5509            self.apply_inverse_row_visibility_event(&item.event);
5510            self.apply_inverse_staged_formula_event(&item.event);
5511        }
5512        if !batch.is_empty() {
5513            let events = batch
5514                .iter()
5515                .map(|item| item.event.clone())
5516                .collect::<Vec<_>>();
5517            self.clear_logged_cell_format_states(&events);
5518        }
5519        self.mirror_undo_batch_to_arrow(&batch);
5520        if !batch.is_empty() {
5521            for item in &batch {
5522                self.record_change_for_event(&item.event);
5523            }
5524        }
5525        crate::engine::trace::fz_event!(
5526            tracing::Level::INFO,
5527            "history",
5528            "history.replay",
5529            op = "undo",
5530            events_replayed = batch.len(),
5531            ownership_resyncs = batch.len()
5532        );
5533        Ok(())
5534    }
5535
5536    pub fn redo_logged(
5537        &mut self,
5538        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5539        log: &mut crate::engine::ChangeLog,
5540    ) -> Result<(), crate::engine::EditorError> {
5541        let pending_events = undo.pending_redo_events();
5542        self.preflight_replay_admission(&pending_events, true)?;
5543        let invalidation_baseline = self.invalidation_baseline();
5544        self.invalidate_for_change_events(
5545            &pending_events,
5546            LoggedEditDirection::ForwardReplay,
5547            invalidation_baseline,
5548        );
5549        let batch = undo.redo(&mut self.graph, log)?;
5550        for item in &batch {
5551            self.apply_forward_row_visibility_event(&item.event);
5552            self.apply_forward_staged_formula_event(&item.event);
5553        }
5554        if !batch.is_empty() {
5555            let events = batch
5556                .iter()
5557                .map(|item| item.event.clone())
5558                .collect::<Vec<_>>();
5559            self.clear_logged_cell_format_states(&events);
5560        }
5561        self.mirror_redo_batch_to_arrow(&batch);
5562        if !batch.is_empty() {
5563            for item in &batch {
5564                self.record_change_for_event(&item.event);
5565            }
5566        }
5567        crate::engine::trace::fz_event!(
5568            tracing::Level::INFO,
5569            "history",
5570            "history.replay",
5571            op = "redo",
5572            events_replayed = batch.len(),
5573            ownership_resyncs = batch.len()
5574        );
5575        Ok(())
5576    }
5577
5578    /// Undo the last committed atomic action using the journal stack.
5579    ///
5580    /// This path does not require a `ChangeLog`.
5581    pub fn undo_action(
5582        &mut self,
5583        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5584    ) -> Result<(), crate::engine::EditorError> {
5585        let Some(journal) = undo.pop_undo_action() else {
5586            return Ok(());
5587        };
5588        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5589            undo.push_done_action(journal);
5590            return Err(error);
5591        }
5592        let invalidation_baseline = self.invalidation_baseline();
5593
5594        self.invalidate_for_action_journal(
5595            &journal,
5596            LoggedEditDirection::InverseReplay,
5597            invalidation_baseline,
5598        );
5599        journal.graph.undo(&mut self.graph)?;
5600        self.apply_inverse_row_visibility_events(&journal.graph.events);
5601        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
5602        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5603            for event in &journal.graph.events {
5604                self.record_change_for_event(event);
5605            }
5606        }
5607
5608        #[cfg(feature = "tracing")]
5609        let events_replayed = journal.graph.events.len();
5610        crate::engine::trace::fz_event!(
5611            tracing::Level::INFO,
5612            "history",
5613            "history.replay",
5614            op = "undo",
5615            events_replayed,
5616            ownership_resyncs = events_replayed
5617        );
5618        undo.push_redo_action(journal);
5619        Ok(())
5620    }
5621
5622    /// Redo the last undone atomic action using the journal stack.
5623    ///
5624    /// This path does not require a `ChangeLog`.
5625    pub fn redo_action(
5626        &mut self,
5627        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5628    ) -> Result<(), crate::engine::EditorError> {
5629        let Some(journal) = undo.pop_redo_action() else {
5630            return Ok(());
5631        };
5632        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5633            undo.push_redo_action(journal);
5634            return Err(error);
5635        }
5636        let invalidation_baseline = self.invalidation_baseline();
5637        self.invalidate_for_action_journal(
5638            &journal,
5639            LoggedEditDirection::ForwardReplay,
5640            invalidation_baseline,
5641        );
5642        journal.graph.redo(&mut self.graph)?;
5643        self.apply_forward_row_visibility_events(&journal.graph.events);
5644        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ false);
5645        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5646            for event in &journal.graph.events {
5647                self.record_change_for_event(event);
5648            }
5649        }
5650
5651        #[cfg(feature = "tracing")]
5652        let events_replayed = journal.graph.events.len();
5653        crate::engine::trace::fz_event!(
5654            tracing::Level::INFO,
5655            "history",
5656            "history.replay",
5657            op = "redo",
5658            events_replayed,
5659            ownership_resyncs = events_replayed
5660        );
5661        undo.push_done_action(journal);
5662        Ok(())
5663    }
5664
5665    fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5666        let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5667        // Coord stores 0-based indices.
5668        let row = addr.coord.row() + 1;
5669        let col = addr.coord.col() + 1;
5670        (sheet, row, col)
5671    }
5672
5673    fn mirror_undo_batch_to_arrow(
5674        &mut self,
5675        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5676    ) {
5677        // Undo applies inverses in reverse order.
5678        for item in batch.iter().rev() {
5679            self.mirror_inverse_change_to_arrow(&item.event);
5680        }
5681    }
5682
5683    fn mirror_redo_batch_to_arrow(
5684        &mut self,
5685        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5686    ) {
5687        // Redo applies events in forward order.
5688        for item in batch.iter() {
5689            self.mirror_forward_change_to_arrow(&item.event);
5690        }
5691    }
5692
5693    fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5694        use crate::engine::ChangeEvent;
5695        use formualizer_common::LiteralValue;
5696
5697        match ev {
5698            ChangeEvent::SetValue {
5699                addr,
5700                old_value,
5701                old_formula,
5702                ..
5703            } => {
5704                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5705                if old_formula.is_some() {
5706                    self.clear_delta_overlay_cell(&sheet, row, col);
5707                } else {
5708                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5709                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5710                }
5711            }
5712            ChangeEvent::SetFormula {
5713                addr,
5714                old_value,
5715                old_formula,
5716                ..
5717            } => {
5718                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5719                if old_formula.is_some() {
5720                    self.clear_delta_overlay_cell(&sheet, row, col);
5721                } else {
5722                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5723                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5724                }
5725            }
5726            ChangeEvent::SpillCommitted { old, new, .. } => {
5727                // Inverse: restore `old` (or clear if none).
5728                self.mirror_spill_snapshot(new, /*clear_only=*/ true);
5729                if let Some(snap) = old {
5730                    self.mirror_spill_snapshot(snap, /*clear_only=*/ false);
5731                }
5732            }
5733            ChangeEvent::SpillCleared { old, .. } => {
5734                // Inverse: restore prior spill.
5735                self.mirror_spill_snapshot(old, /*clear_only=*/ false);
5736            }
5737            ChangeEvent::SetRowVisibility { .. } => {
5738                // Engine-side metadata only; no Arrow overlay effect.
5739            }
5740            _ => {}
5741        }
5742    }
5743
5744    fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5745        use crate::engine::ChangeEvent;
5746
5747        match ev {
5748            ChangeEvent::SetValue { addr, new, .. } => {
5749                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5750                self.mirror_value_to_overlay(&sheet, row, col, new);
5751            }
5752            ChangeEvent::SetFormula { addr, .. } => {
5753                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5754                self.clear_delta_overlay_cell(&sheet, row, col);
5755                // Keep any computed overlay for this cell as-is; it will be recomputed on demand.
5756            }
5757            ChangeEvent::SpillCommitted { old, new, .. } => {
5758                if let Some(snap) = old {
5759                    self.mirror_spill_snapshot(snap, /*clear_only=*/ true);
5760                }
5761                self.mirror_spill_snapshot(new, /*clear_only=*/ false);
5762            }
5763            ChangeEvent::SpillCleared { old, .. } => {
5764                self.mirror_spill_snapshot(old, /*clear_only=*/ true);
5765            }
5766            ChangeEvent::SetRowVisibility { .. } => {
5767                // Engine-side metadata only; no Arrow overlay effect.
5768            }
5769            _ => {
5770                // Other graph structural operations do not have direct value effects in Arrow.
5771            }
5772        }
5773    }
5774
5775    fn mirror_spill_snapshot(
5776        &mut self,
5777        snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5778        clear_only: bool,
5779    ) {
5780        use formualizer_common::LiteralValue;
5781
5782        let mut i = 0usize;
5783        for row in &snap.values {
5784            for v in row {
5785                if let Some(cell) = snap.target_cells.get(i) {
5786                    let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5787                    let out = if clear_only {
5788                        LiteralValue::Empty
5789                    } else {
5790                        v.clone()
5791                    };
5792                    self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5793                }
5794                i += 1;
5795            }
5796        }
5797        // If target_cells is longer than values (should not happen), clear remaining cells.
5798        if clear_only {
5799            for cell in snap.target_cells.iter().skip(i) {
5800                let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5801                self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5802            }
5803        }
5804    }
5805
5806    pub fn set_default_sheet_by_name(&mut self, name: &str) {
5807        self.graph.set_default_sheet_by_name(name);
5808    }
5809
5810    pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5811        self.graph.set_default_sheet_by_id(id);
5812    }
5813
5814    pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5815        self.graph.set_sheet_index_mode(mode);
5816    }
5817
5818    #[cfg(feature = "test-support")]
5819    #[doc(hidden)]
5820    pub fn mark_all_formulas_dirty_for_test(&mut self) {
5821        self.mark_all_formula_vertices_dirty();
5822    }
5823
5824    #[cfg(feature = "test-support")]
5825    #[doc(hidden)]
5826    pub fn take_criteria_mask_work_for_test() -> (usize, usize) {
5827        criteria_mask_test_hooks::take_mask_work()
5828    }
5829
5830    #[cfg(feature = "test-support")]
5831    #[doc(hidden)]
5832    pub fn lookup_index_cache_report_for_test(&self) -> LookupIndexCacheReport {
5833        self.lookup_index_cache.report()
5834    }
5835
5836    #[cfg(any(test, feature = "benchmark_internal"))]
5837    #[doc(hidden)]
5838    pub fn reset_recalc_reuse_probe(&mut self) {
5839        *self.recalc_reuse_probe.get_mut().unwrap() = RecalcReuseProbe::default();
5840    }
5841
5842    #[cfg(any(test, feature = "benchmark_internal"))]
5843    #[doc(hidden)]
5844    pub fn recalc_reuse_probe(&self) -> RecalcReuseProbe {
5845        let mut probe = self.recalc_reuse_probe.lock().unwrap().clone();
5846        if let Some(cached) = self.cached_static_schedule.as_ref() {
5847            let entry_bytes = |e: &CachedScheduleEntry| {
5848                std::mem::size_of::<CachedScheduleEntry>()
5849                    + e.candidate_vertices.heap_bytes()
5850                    + std::mem::size_of::<crate::engine::Schedule>()
5851                    + 2 * std::mem::size_of::<usize>()
5852                    + schedule_probe_retained_bytes(&e.schedule)
5853            };
5854            probe.schedule_retained_bytes = entry_bytes(cached)
5855                + self.recent_schedules.iter().map(entry_bytes).sum::<usize>()
5856                + self.base_schedule.as_ref().map_or(0, entry_bytes)
5857                + self.recent_schedules.capacity() * std::mem::size_of::<CachedScheduleEntry>();
5858        }
5859        probe
5860    }
5861
5862    #[cfg(test)]
5863    pub(crate) fn cached_static_schedule_for_test(&self) -> Option<Arc<crate::engine::Schedule>> {
5864        self.cached_static_schedule
5865            .as_ref()
5866            .map(|cached| Arc::clone(&cached.schedule))
5867    }
5868
5869    fn clear_cached_static_schedule(&mut self) {
5870        self.cached_static_schedule = None;
5871        self.recent_schedules.clear();
5872        self.base_schedule = None;
5873    }
5874
5875    /// Keep a replaced schedule among the recent ones when it is still
5876    /// current and small; drop stale ones and the oldest beyond the bounds.
5877    fn retain_recent_schedule(&mut self, entry: CachedScheduleEntry) {
5878        const RECENT_SCHEDULES: usize = 8;
5879        const RECENT_SCHEDULE_VERTICES: usize = 65_536;
5880        let revision = self.schedule_cache_authority_revision();
5881        let epoch = self.topology_epoch;
5882        self.recent_schedules
5883            .retain(|e| e.topology_epoch == epoch && e.authority_revision == revision);
5884        let current =
5885            |e: &CachedScheduleEntry| e.topology_epoch == epoch && e.authority_revision == revision;
5886        if self.base_schedule.as_ref().is_some_and(|b| !current(b)) {
5887            self.base_schedule = None;
5888        }
5889        // The largest current schedule becomes the base; a replaced base
5890        // may still join the recent ones.
5891        let mut entry = entry;
5892        if current(&entry)
5893            && entry.candidate_vertices.len() > BASE_SCHEDULE_MIN_VERTICES
5894            && self
5895                .base_schedule
5896                .as_ref()
5897                .is_none_or(|b| entry.candidate_vertices.len() > b.candidate_vertices.len())
5898        {
5899            match self.base_schedule.replace(entry) {
5900                Some(previous) => entry = previous,
5901                None => return,
5902            }
5903        }
5904        if entry.topology_epoch != epoch
5905            || entry.authority_revision != revision
5906            || entry.candidate_vertices.len() > RECENT_SCHEDULE_VERTICES / 4
5907        {
5908            return;
5909        }
5910        self.recent_schedules.push(entry);
5911        let mut total: usize = self
5912            .recent_schedules
5913            .iter()
5914            .map(|e| e.candidate_vertices.len())
5915            .sum();
5916        while self.recent_schedules.len() > RECENT_SCHEDULES || total > RECENT_SCHEDULE_VERTICES {
5917            let oldest = self.recent_schedules.remove(0);
5918            total -= oldest.candidate_vertices.len();
5919        }
5920    }
5921
5922    fn invalidation_baseline(&self) -> InvalidationBaseline {
5923        InvalidationBaseline {
5924            snapshot_id: self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed),
5925            topology_epoch: self.topology_epoch,
5926        }
5927    }
5928
5929    fn classify_change_events(
5930        events: &[crate::engine::ChangeEvent],
5931        direction: LoggedEditDirection,
5932    ) -> LoggedEditImpact {
5933        use crate::engine::ChangeEvent;
5934
5935        events
5936            .iter()
5937            .map(|event| match event {
5938                ChangeEvent::CompoundStart { .. } | ChangeEvent::CompoundEnd { .. } => {
5939                    LoggedEditImpact::NoOp
5940                }
5941                ChangeEvent::SetRowVisibility { .. }
5942                | ChangeEvent::SetValue {
5943                    old_formula: None,
5944                    old_value: Some(_),
5945                    ..
5946                } => LoggedEditImpact::DataOnly,
5947                // Original canonical writes can lack an Arrow old value while
5948                // updating an existing placeholder. Inverse replay actually
5949                // removes that vertex; a later redo recreates it.
5950                ChangeEvent::SetValue {
5951                    old_formula: None,
5952                    old_value: None,
5953                    ..
5954                } if direction == LoggedEditDirection::Original => LoggedEditImpact::DataOnly,
5955                ChangeEvent::SetValue { .. }
5956                | ChangeEvent::SetFormula { .. }
5957                | ChangeEvent::AddVertex { .. }
5958                | ChangeEvent::RemoveVertex { .. }
5959                | ChangeEvent::VertexMoved { .. }
5960                | ChangeEvent::FormulaAdjusted { .. }
5961                | ChangeEvent::NamedRangeAdjusted { .. }
5962                | ChangeEvent::EdgeAdded { .. }
5963                | ChangeEvent::EdgeRemoved { .. }
5964                | ChangeEvent::DefineName { .. }
5965                | ChangeEvent::UpdateName { .. }
5966                | ChangeEvent::DeleteName { .. }
5967                | ChangeEvent::SpillCommitted { .. }
5968                | ChangeEvent::SpillCleared { .. }
5969                | ChangeEvent::StagedFormulaCellChanged { .. } => LoggedEditImpact::Topology,
5970            })
5971            .max()
5972            .unwrap_or(LoggedEditImpact::NoOp)
5973    }
5974
5975    fn classify_arrow_undo(arrow: &crate::engine::ArrowUndoBatch) -> LoggedEditImpact {
5976        use crate::engine::ArrowOp;
5977
5978        arrow
5979            .ops
5980            .iter()
5981            .map(|op| match op {
5982                ArrowOp::SetDeltaCell { .. } | ArrowOp::SetComputedCell { .. } => {
5983                    LoggedEditImpact::DataOnly
5984                }
5985                ArrowOp::RestoreComputedRect { .. }
5986                | ArrowOp::InsertRows { .. }
5987                | ArrowOp::InsertCols { .. } => LoggedEditImpact::Topology,
5988            })
5989            .max()
5990            .unwrap_or(LoggedEditImpact::NoOp)
5991    }
5992
5993    fn apply_logged_edit_impact(
5994        &mut self,
5995        impact: LoggedEditImpact,
5996        baseline: InvalidationBaseline,
5997    ) {
5998        match impact {
5999            LoggedEditImpact::NoOp => {}
6000            LoggedEditImpact::DataOnly => {
6001                if self.topology_epoch == baseline.topology_epoch
6002                    && self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
6003                        == baseline.snapshot_id
6004                {
6005                    self.mark_data_edited();
6006                }
6007            }
6008            LoggedEditImpact::Topology => {
6009                // Some structural entry points already publish topology
6010                // invalidation. Do not bump the same batch twice.
6011                if self.topology_epoch == baseline.topology_epoch {
6012                    self.mark_topology_edited();
6013                }
6014            }
6015        }
6016    }
6017
6018    fn invalidate_for_change_events(
6019        &mut self,
6020        events: &[crate::engine::ChangeEvent],
6021        direction: LoggedEditDirection,
6022        baseline: InvalidationBaseline,
6023    ) {
6024        self.apply_logged_edit_impact(Self::classify_change_events(events, direction), baseline);
6025    }
6026
6027    fn invalidate_for_action_journal(
6028        &mut self,
6029        journal: &crate::engine::ActionJournal,
6030        direction: LoggedEditDirection,
6031        baseline: InvalidationBaseline,
6032    ) {
6033        let impact = Self::classify_change_events(&journal.graph.events, direction)
6034            .max(Self::classify_arrow_undo(&journal.arrow));
6035        self.apply_logged_edit_impact(impact, baseline);
6036    }
6037
6038    /// Mark data edited: bump snapshot and set edited flag.
6039    /// Value-only edits keep the stable-topology schedule cache alive.
6040    pub fn mark_data_edited(&mut self) {
6041        self.lookup_index_cache.clear();
6042        self.snapshot_id
6043            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6044        self.has_edited = true;
6045    }
6046
6047    /// Mark a topology-changing edit: bump snapshot + topology epoch and invalidate cached schedules.
6048    pub fn mark_topology_edited(&mut self) {
6049        self.arrow_sheet_index.take();
6050        self.lookup_index_cache.clear();
6051        self.snapshot_id
6052            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6053        self.topology_epoch = self.topology_epoch.wrapping_add(1);
6054        self.graph.bump_topology_revision();
6055        self.clear_cached_static_schedule();
6056        if let Some(ledger) = self.active_resource_ledger.as_mut() {
6057            let released = ledger.account_mixed_cache(0);
6058            debug_assert!(released.is_ok());
6059        }
6060        self.has_edited = true;
6061        // Eager sync at a topology edit, so read-only (`&self`) plans and
6062        // inspection see a current authority (not during a load or an open
6063        // structural capture).
6064        self.graph.authority_sync_eager();
6065    }
6066
6067    fn mark_all_formula_vertices_dirty(&mut self) {
6068        let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
6069        for vertex in vertices {
6070            self.graph.mark_vertex_dirty(vertex);
6071        }
6072    }
6073
6074    fn mark_moved_formula_vertices_dirty(
6075        &mut self,
6076        summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
6077    ) {
6078        for vertex in &summary.vertices_moved {
6079            if self.graph.has_formula(*vertex) {
6080                self.graph.mark_vertex_dirty(*vertex);
6081            }
6082        }
6083    }
6084
6085    /// Access Arrow sheet store (read-only)
6086    pub fn sheet_store(&self) -> &SheetStore {
6087        &self.arrow_sheets
6088    }
6089
6090    /// True when any sheet carries manual/filter row-visibility state.
6091    /// Used by load-time freshness checks (see `Engine::adopt_file_sheets`).
6092    pub(crate) fn has_row_visibility_state(&self) -> bool {
6093        !self.row_visibility.is_empty()
6094    }
6095
6096    /// Access Arrow sheet store (mutable)
6097    pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
6098        self.arrow_sheet_index.take();
6099        &mut self.arrow_sheets
6100    }
6101
6102    pub fn has_staged_formulas(&self) -> bool {
6103        !self.staged_formulas.is_empty()
6104    }
6105
6106    pub fn staged_formula_count(&self) -> usize {
6107        self.staged_formulas.values().map(StagedSheet::len).sum()
6108    }
6109
6110    /// Stage a formula text instead of inserting into the graph (used when deferring is enabled).
6111    pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
6112        self.staged_formulas
6113            .entry(sheet.to_string())
6114            .or_default()
6115            .stage(row, col, text);
6116        self.staged_formula_index.stage(sheet, row, col);
6117        if let Some(sheet) = self.graph.sheet_id(sheet) {
6118            self.invalidate_pending_spills(StructuralScope::Cell {
6119                sheet,
6120                row: row.saturating_sub(1),
6121                col: col.saturating_sub(1),
6122            });
6123        }
6124    }
6125
6126    fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
6127        for (row, col, _) in &staged.entries {
6128            self.staged_formula_index.remove(sheet, *row, *col);
6129        }
6130        if staged.deferred_package.is_some() {
6131            self.staged_formula_index.set_package(sheet, None);
6132        }
6133    }
6134
6135    fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
6136        self.staged_formulas.insert(sheet, staged);
6137    }
6138
6139    #[doc(hidden)]
6140    pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
6141        SourceFormulaIngress { engine: self }
6142    }
6143
6144    #[doc(hidden)]
6145    /// Test-only fault-injection seam. Not part of the supported API; it exists so
6146    /// integration tests in sibling crates can fail a commit at an exact point.
6147    #[doc(hidden)]
6148    #[cfg(any(test, feature = "test-support"))]
6149    pub fn set_before_prepared_span_commit_hook(
6150        &mut self,
6151        hook: impl FnOnce() + Send + Sync + 'static,
6152    ) {
6153        self.before_prepared_span_commit_hook = Some(Box::new(hook));
6154    }
6155
6156    #[doc(hidden)]
6157    /// Test-only fault-injection seam, matching `set_after_eager_proposal_commit_hook`.
6158    #[cfg(test)]
6159    pub(crate) fn set_before_target_preparation_commit_hook(
6160        &mut self,
6161        hook: impl FnOnce() + Send + Sync + 'static,
6162    ) {
6163        self.before_target_preparation_commit_hook = Some(Box::new(hook));
6164    }
6165
6166    fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
6167        if let Ok(replay) = package.replay.lock()
6168            && let Some(footprint) = replay.selection_cache_footprint()
6169            && !self
6170                .source_cache_footprints
6171                .iter()
6172                .any(|known| known.ptr_eq(&footprint))
6173        {
6174            self.source_cache_footprints.push(footprint);
6175        }
6176        let sheet = package.sheet_name.clone();
6177        let staged = self.staged_formulas.entry(sheet.clone()).or_default();
6178        debug_assert!(staged.deferred_package.is_none());
6179        staged.deferred_package = Some(package);
6180        staged.reconcile_attached_deferred_package();
6181        self.staged_formula_index
6182            .set_package(&sheet, staged.deferred_package.as_ref());
6183        if let Some(sheet_id) = self.graph.sheet_id(&sheet) {
6184            let package = self
6185                .staged_formulas
6186                .get(&sheet)
6187                .and_then(|staged| staged.deferred_package.as_ref());
6188            for &(vertex, anchor, region) in &self.blocked_pending_spills {
6189                if anchor.sheet_id == sheet_id
6190                    && self.graph.vertex_exists(vertex)
6191                    && self.graph.get_cell_ref(vertex) == Some(anchor)
6192                    && self.staged_formula_index.package_occupies_spill(
6193                        &sheet,
6194                        (anchor.coord.row() + 1, anchor.coord.col() + 1),
6195                        (
6196                            region.rows.query_bounds().1 + 1,
6197                            region.cols.query_bounds().1 + 1,
6198                        ),
6199                        |point| package.is_some_and(|package| package.suppressed.contains(&point)),
6200                    )
6201                {
6202                    self.graph.mark_vertex_dirty(vertex);
6203                }
6204            }
6205        }
6206    }
6207
6208    pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
6209        let mut removed = None;
6210        let mut remove_sheet = false;
6211        let mut had_package = false;
6212        if let Some(entries) = self.staged_formulas.get_mut(sheet) {
6213            had_package = entries.deferred_package.is_some();
6214            removed = entries.remove(row, col);
6215            remove_sheet = entries.is_empty();
6216        }
6217        let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
6218        if !ordinary_removed && had_package {
6219            self.staged_formula_index.touch_package(sheet);
6220        }
6221        if remove_sheet {
6222            self.staged_formulas.remove(sheet);
6223            self.staged_formula_index.set_package(sheet, None);
6224        }
6225        if (ordinary_removed || had_package)
6226            && let Some(sheet) = self.graph.sheet_id(sheet)
6227        {
6228            self.invalidate_pending_spills(StructuralScope::Cell {
6229                sheet,
6230                row: row.saturating_sub(1),
6231                col: col.saturating_sub(1),
6232            });
6233        }
6234        removed
6235    }
6236
6237    pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
6238        if self.staged_formulas.remove(sheet).is_some() {
6239            self.staged_formula_index.clear_sheet(sheet);
6240            if let Some(sheet_id) = self.graph.sheet_id(sheet) {
6241                self.invalidate_pending_spills(StructuralScope::Sheet(sheet_id));
6242            }
6243        }
6244    }
6245
6246    pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
6247        let Some(entries) = self.staged_formulas.remove(old) else {
6248            return;
6249        };
6250        self.staged_formula_index.clear_sheet(old);
6251        let (formulas, mut package) = entries.into_parts();
6252        for (row, col, text) in formulas {
6253            self.stage_formula_text(new, row, col, text);
6254        }
6255        if let Some(package) = package.as_mut() {
6256            package.sheet_name = new.to_string();
6257        }
6258        if let Some(package) = package {
6259            self.stage_deferred_formula_package(package);
6260        }
6261    }
6262
6263    /// Get a staged formula text for a given cell if present (cloned).
6264    pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
6265        self.staged_formulas
6266            .get(sheet)
6267            .and_then(|v| v.get(row, col))
6268    }
6269
6270    pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
6271        &self.formula_parse_diagnostics
6272    }
6273
6274    pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
6275        std::mem::take(&mut self.formula_parse_diagnostics)
6276    }
6277
6278    pub fn clear_formula_parse_diagnostics(&mut self) {
6279        self.formula_parse_diagnostics.clear();
6280    }
6281
6282    pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
6283        self.last_formula_ingest_report.as_ref()
6284    }
6285
6286    pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
6287        &self.formula_ingest_report_total
6288    }
6289
6290    #[cfg(test)]
6291    pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
6292        &mut self,
6293        hook: impl FnOnce() + Send + Sync + 'static,
6294    ) {
6295        self.before_target_planning_snapshot_hook = Some(Box::new(hook));
6296    }
6297
6298    #[cfg(test)]
6299    pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
6300        self.inject_target_semantic_stale_once_for_test = true;
6301    }
6302
6303    #[cfg(test)]
6304    pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
6305        self.force_virtual_dep_changes_remaining_for_test = rounds;
6306    }
6307
6308    #[cfg(test)]
6309    pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
6310        self.fail_evaluation_commit_preflight_once_for_test = true;
6311    }
6312
6313    #[cfg(test)]
6314    pub(crate) fn set_target_preparation_fault_for_test(
6315        &mut self,
6316        fault: crate::engine::target_preparation::TargetPreparationFault,
6317    ) {
6318        self.target_preparation_fault_for_test = Some(fault);
6319    }
6320
6321    #[cfg(test)]
6322    pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
6323        self.staged_formula_index.revision()
6324    }
6325
6326    #[cfg(test)]
6327    pub(crate) fn deferred_package_for_test(
6328        &self,
6329        sheet: &str,
6330    ) -> &crate::engine::DeferredFormulaPackage {
6331        self.staged_formulas
6332            .get(sheet)
6333            .unwrap()
6334            .deferred_package
6335            .as_ref()
6336            .unwrap()
6337    }
6338
6339    #[cfg(test)]
6340    pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
6341        let ordinary_storage = self
6342            .staged_formulas
6343            .values()
6344            .map(|sheet| sheet.entries.len())
6345            .sum::<usize>();
6346        let package_storage = self
6347            .staged_formulas
6348            .values()
6349            .filter(|sheet| sheet.deferred_package.is_some())
6350            .count();
6351        ordinary_storage == self.staged_formula_index.ordinary_count()
6352            && package_storage == self.staged_formula_index.package_count()
6353            && self.staged_formulas.iter().all(|(name, sheet)| {
6354                sheet.entries.iter().all(|(row, col, _)| {
6355                    let leases = self
6356                        .staged_formula_index
6357                        .leases_in_region(name, *row, *col, *row, *col);
6358                    leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
6359                })
6360            })
6361    }
6362
6363    #[cfg(test)]
6364    pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
6365        self.evaluation_request_begin_count_for_test
6366    }
6367
6368    #[cfg(test)]
6369    pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
6370        &mut self,
6371        hook: impl FnOnce() + Send + Sync + 'static,
6372    ) {
6373        self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
6374    }
6375
6376    #[cfg(test)]
6377    pub(crate) fn set_after_eager_proposal_commit_hook(
6378        &mut self,
6379        hook: impl FnOnce() + Send + Sync + 'static,
6380    ) {
6381        self.after_eager_proposal_commit_hook = Some(Box::new(hook));
6382    }
6383
6384    #[cfg(test)]
6385    pub(crate) fn topology_epoch_for_test(&self) -> u64 {
6386        self.topology_epoch
6387    }
6388
6389    #[cfg(test)]
6390    pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
6391        self.graph.topology_revision()
6392    }
6393
6394    fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
6395        self.formula_ingest_report_total.mode = report.mode;
6396        self.formula_ingest_report_total.accumulate(&report);
6397        self.last_formula_ingest_report = Some(report);
6398    }
6399
6400    fn collect_planning_function_requests(
6401        ast: &ASTNode,
6402        requests: &mut Vec<(String, String, usize)>,
6403    ) {
6404        match &ast.node_type {
6405            ASTNodeType::Function { name, args } => {
6406                requests.push((String::new(), name.clone(), args.len()));
6407                for arg in args {
6408                    Self::collect_planning_function_requests(arg, requests);
6409                }
6410            }
6411            ASTNodeType::BinaryOp { left, right, .. } => {
6412                Self::collect_planning_function_requests(left, requests);
6413                Self::collect_planning_function_requests(right, requests);
6414            }
6415            ASTNodeType::UnaryOp { expr, .. } => {
6416                Self::collect_planning_function_requests(expr, requests);
6417            }
6418            ASTNodeType::Call { callee, args } => {
6419                Self::collect_planning_function_requests(callee, requests);
6420                for arg in args {
6421                    Self::collect_planning_function_requests(arg, requests);
6422                }
6423            }
6424            ASTNodeType::Array(rows) => {
6425                for cell in rows.iter().flatten() {
6426                    Self::collect_planning_function_requests(cell, requests);
6427                }
6428            }
6429            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6430        }
6431    }
6432
6433    fn prepared_function_semantics_changed(
6434        &self,
6435        preparation: &crate::engine::FormulaCompressedPreparation,
6436        guard: &crate::function_registry::SemanticEpochReadGuard,
6437    ) -> bool {
6438        if preparation.function_semantic_epoch == guard.epoch() {
6439            return false;
6440        }
6441
6442        guard
6443            .semantic_changes_affect_requests_since(preparation.function_semantic_epoch, Vec::new())
6444    }
6445
6446    pub(crate) fn prepare_source_formula_families(
6447        &mut self,
6448        sheet_name: &str,
6449        families: &[crate::engine::SourceFormulaFamily],
6450    ) -> crate::engine::FormulaCompressedPreparation {
6451        crate::engine::FormulaCompressedPreparation {
6452            engine_token: Arc::clone(&self.source_formula_token),
6453            function_semantic_epoch: crate::function_registry::semantic_epoch(),
6454            function_provider_revision: None,
6455            function_semantics_used: false,
6456            sheet_name: Arc::from(sheet_name),
6457            rejected: BTreeMap::new(),
6458            eager_replay: Vec::new(),
6459            preparation_spool_replays: 0,
6460            clean_rejected_anchor_counts: [0; 3],
6461            fragmented_rejected_anchor_counts: [0; 3],
6462            exact_replay: None,
6463            replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6464        }
6465    }
6466
6467    fn prepare_source_formula_proposals(
6468        &mut self,
6469        sheet_name: &str,
6470        families: &[crate::engine::SourceFormulaFamily],
6471        authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6472        replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6473        formula_record_count: u64,
6474        replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6475        suppressed: &BTreeSet<(u32, u32)>,
6476        consumed: &FxHashSet<(crate::engine::SourceFamilyId, crate::engine::SourceCoord)>,
6477        consumed_engine: Option<&Arc<()>>,
6478    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
6479        let mut preparation = self.prepare_source_formula_families(sheet_name, families);
6480        preparation.exact_replay = Some(Arc::clone(&replay));
6481        preparation
6482            .replay_disposition
6483            .register_consumed_members(consumed_engine, consumed.iter().copied());
6484        if !preparation
6485            .replay_disposition
6486            .consumed_engine_matches(&self.source_formula_token)
6487        {
6488            return Err(ExcelError::new(ExcelErrorKind::Value)
6489                .with_message("ResidualConsumedEngineMismatch"));
6490        }
6491        preparation
6492            .replay_disposition
6493            .extend_suppressed_excel_coords(suppressed.iter().copied());
6494        Ok(preparation)
6495    }
6496
6497    fn formula_batch_from_exact_replay(
6498        &mut self,
6499        sheet_name: &str,
6500        replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
6501    ) -> Result<FormulaIngestBatch, ExcelError> {
6502        let mut cache = rustc_hash::FxHashMap::default();
6503        let mut formulas = Vec::new();
6504        for record in replayed {
6505            let key = if record.text.starts_with('=') {
6506                record.text
6507            } else {
6508                format!("={}", record.text)
6509            };
6510            let ast_id = if let Some(cached) = cache.get(&key) {
6511                *cached
6512            } else {
6513                let parsed = match formualizer_parse::parser::parse(&key) {
6514                    Ok(parsed) => parsed,
6515                    Err(error) => {
6516                        let Some(parsed) = self.handle_formula_parse_error(
6517                            sheet_name,
6518                            record.row,
6519                            record.col,
6520                            &key,
6521                            error.to_string(),
6522                        )?
6523                        else {
6524                            continue;
6525                        };
6526                        parsed
6527                    }
6528                };
6529                let ast_id = self.intern_formula_ast(&parsed);
6530                cache.insert(key.clone(), ast_id);
6531                ast_id
6532            };
6533            formulas.push(
6534                FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
6535                    .with_source_proof(record.source_order, record.family, record.partition_owner),
6536            );
6537        }
6538        Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
6539    }
6540
6541    fn prepare_target_combined_legacy_graph(
6542        &self,
6543        packages: &[PreparedTargetSourcePackage],
6544        ordinary: &[PreparedOrdinaryStagedFormula],
6545    ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
6546        let mut planned_by_coord = BTreeMap::new();
6547        for package in packages {
6548            for (row, col, ast_id, plan) in &package.legacy {
6549                planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
6550            }
6551        }
6552        for formula in ordinary {
6553            if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
6554                planned_by_coord.insert(
6555                    (formula.sheet_id, formula.lease.row, formula.lease.col),
6556                    (ast_id, plan),
6557                );
6558            }
6559        }
6560        let planned = planned_by_coord
6561            .into_iter()
6562            .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
6563            .collect::<Vec<_>>();
6564        let formula_count = planned.len();
6565        let graph = self
6566            .graph
6567            .prepare_legacy_graph_plan_multi_sheet(planned)
6568            .map_err(|error| {
6569                ExcelError::new(ExcelErrorKind::Value)
6570                    .with_message(format!("target graph preparation failed: {error}"))
6571            })?;
6572        Ok((graph, formula_count))
6573    }
6574
6575    fn replay_target_coordinates(
6576        &mut self,
6577        replay: &Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6578        coordinates: &[(u32, u32)],
6579        deadline: Option<std::time::Instant>,
6580        scratch: &mut u64,
6581    ) -> Result<Option<Vec<crate::engine::DeferredReplayFormula>>, ExcelError> {
6582        if coordinates.is_empty() {
6583            return Ok(Some(Vec::new()));
6584        }
6585        let mut guard = replay.lock().map_err(|_| {
6586            ExcelError::new(ExcelErrorKind::Value)
6587                .with_message("deferred formula spool lock poisoned")
6588        })?;
6589        let retained = guard.selection_cache_footprint().is_some();
6590        let records = guard.replay_selected_exact(coordinates, &mut |work, bytes| {
6591            self.target_preparation_checkpoint(deadline, work)?;
6592            if retained && let Some(ledger) = self.active_resource_ledger.as_mut() {
6593                ledger
6594                    .reserve_retained(bytes)
6595                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
6596                self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
6597            }
6598            self.reserve_graph_source_scratch(bytes)?;
6599            *scratch = scratch.saturating_add(bytes);
6600            Ok(())
6601        });
6602        drop(guard);
6603        let reconciled = self.reconcile_source_cache_footprints();
6604        let records = records?;
6605        reconciled?;
6606        Ok(records)
6607    }
6608
6609    fn prepare_target_exact_source_selection(
6610        &mut self,
6611        sheet: &str,
6612        lease: StagedPackageLease,
6613        coordinates: Vec<(u32, u32)>,
6614        previous: &BTreeSet<(u32, u32)>,
6615        allow_partial_shared: bool,
6616        deadline: Option<std::time::Instant>,
6617        scratch: &mut u64,
6618    ) -> Result<Option<PreparedTargetSourcePackage>, ExcelError> {
6619        let package = self
6620            .staged_formulas
6621            .get(sheet)
6622            .and_then(|s| s.deferred_package.as_ref())
6623            .unwrap();
6624        if !package.source_geometry_complete
6625            || ((!package.families.is_empty() || !package.partitioned_families.is_empty())
6626                && !package.coordinates_cover_families)
6627            || package.reconciliation_replay.is_some()
6628        {
6629            return Ok(None);
6630        }
6631        let selected_points: BTreeSet<_> = coordinates
6632            .into_iter()
6633            .filter(|point| !package.suppressed.contains(point))
6634            .collect();
6635        let mut prepared = PreparedTargetSourcePackage::empty_selection(
6636            sheet,
6637            self.graph.sheet_id(sheet).unwrap(),
6638            lease,
6639        );
6640        if selected_points.is_empty() {
6641            return Ok(Some(prepared));
6642        }
6643        let mut routing = crate::engine::FormulaReplayDisposition::default();
6644        for partition in &package.partitioned_families {
6645            routing
6646                .register_partition(partition, false)
6647                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
6648        }
6649        routing.extend_suppressed_excel_coords(package.suppressed.iter().copied());
6650        routing.register_consumed_members(
6651            package.consumed_engine.as_ref(),
6652            package.consumed_members.iter().copied(),
6653        );
6654        let replay = Arc::clone(&package.replay);
6655        let source_report = package.accounting_report();
6656        prepared.source_report = source_report;
6657        prepared.disposition = routing.clone();
6658
6659        let replay_points: BTreeSet<_> = selected_points
6660            .iter()
6661            .copied()
6662            .filter(|&(row, col)| !prepared.direct_contains(row, col))
6663            .collect();
6664        let coordinates: Vec<_> = replay_points.iter().copied().collect();
6665        let records = self.replay_target_coordinates(&replay, &coordinates, deadline, scratch)?;
6666        prepared.spool_replays = u64::from(!coordinates.is_empty());
6667        let Some(mut replay_records) = records else {
6668            return Ok(None);
6669        };
6670        for record in &mut replay_records {
6671            if record.family.is_none() {
6672                record.partition_owner = routing
6673                    .ordinary_disposition(crate::engine::SourceCoord {
6674                        row: record.row.saturating_sub(1),
6675                        col: record.col.saturating_sub(1),
6676                    })
6677                    .1;
6678            }
6679        }
6680        replay_records.sort_by_key(|record| record.source_order);
6681        if replay_records
6682            .iter()
6683            .any(|record| !selected_points.contains(&(record.row, record.col)))
6684            || replay_records
6685                .windows(2)
6686                .any(|records| records[0].source_order == records[1].source_order)
6687        {
6688            return Err(ExcelError::new(ExcelErrorKind::Value)
6689                .with_message("invalid indexed exact source selection"));
6690        }
6691        let represented: BTreeSet<_> = replay_records.iter().map(|r| (r.row, r.col)).collect();
6692        if represented != replay_points {
6693            return Err(ExcelError::new(ExcelErrorKind::Value)
6694                .with_message("incomplete indexed exact source selection"));
6695        }
6696        // The last source record must agree with compressed ownership evidence.
6697        // Earlier overridden records retain ordering but demand no dependencies.
6698        let source = self
6699            .staged_formulas
6700            .get(sheet)
6701            .unwrap()
6702            .deferred_package
6703            .as_ref()
6704            .unwrap();
6705        let contains = |rect: crate::engine::SourceRect, coord: crate::engine::SourceCoord| {
6706            coord.row >= rect.start.row
6707                && coord.row <= rect.end.row
6708                && coord.col >= rect.start.col
6709                && coord.col <= rect.end.col
6710        };
6711        let mut checked = BTreeSet::new();
6712        for record in replay_records.iter().rev() {
6713            if !checked.insert((record.row, record.col)) {
6714                continue;
6715            }
6716            let coord = crate::engine::SourceCoord {
6717                row: record.row - 1,
6718                col: record.col - 1,
6719            };
6720            let agrees = |owner, shared: bool| {
6721                record.family == shared.then_some(owner)
6722                    && record.partition_owner.or(record.family) == Some(owner)
6723            };
6724            let mut valid = true;
6725            for family in &source.families {
6726                let owns = match &family.members {
6727                    crate::engine::SourceFamilyMembers::CompleteDomain(domain) => {
6728                        contains(domain.rect(), coord)
6729                    }
6730                    crate::engine::SourceFamilyMembers::ExplicitMembers(members) => {
6731                        members.as_slice().binary_search(&coord).is_ok()
6732                    }
6733                };
6734                if owns && !agrees(family.source_id, true) {
6735                    valid = false;
6736                }
6737            }
6738            for family in &source.partitioned_families {
6739                if family
6740                    .fragments
6741                    .iter()
6742                    .any(|fragment| contains(fragment.rect(), coord))
6743                    && !agrees(family.source_id, true)
6744                {
6745                    valid = false;
6746                }
6747                if let Some(member) = family
6748                    .legacy_members
6749                    .as_slice()
6750                    .iter()
6751                    .find(|member| member.coord == coord)
6752                    && !agrees(
6753                        family.source_id,
6754                        member.kind == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember,
6755                    )
6756                {
6757                    valid = false;
6758                }
6759            }
6760            if !valid {
6761                return Err(ExcelError::new(ExcelErrorKind::Value)
6762                    .with_message("indexed source ownership mismatch"));
6763            }
6764        }
6765        prepared.selected_points = Some(selected_points);
6766        prepared.replay_records = replay_records;
6767        Ok(Some(prepared))
6768    }
6769
6770    fn prepare_target_source_package(
6771        &mut self,
6772        sheet: &str,
6773        lease: StagedPackageLease,
6774        deadline: Option<std::time::Instant>,
6775    ) -> Result<PreparedTargetSourcePackage, ExcelError> {
6776        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
6777            ExcelError::new(ExcelErrorKind::Ref)
6778                .with_message(format!("deferred source sheet not found: {sheet}"))
6779        })?;
6780        let (
6781            source_report,
6782            families,
6783            partitions,
6784            replay,
6785            invalidated,
6786            suppressed,
6787            consumed_members,
6788            consumed_engine,
6789            reconciliation_replay,
6790        ) = {
6791            let package = self
6792                .staged_formulas
6793                .get(sheet)
6794                .and_then(|staged| staged.deferred_package.as_ref())
6795                .ok_or_else(|| {
6796                    ExcelError::new(ExcelErrorKind::Value)
6797                        .with_message("staged deferred source package is unavailable")
6798                })?;
6799            if package.sheet_name != sheet {
6800                return Err(ExcelError::new(ExcelErrorKind::Value)
6801                    .with_message("deferred formula package sheet mismatch"));
6802            }
6803            (
6804                package.accounting_report(),
6805                package.families.clone(),
6806                package.partitioned_families.clone(),
6807                Arc::clone(&package.replay),
6808                package.invalidated.clone(),
6809                package.suppressed.clone(),
6810                package.consumed_members.clone(),
6811                package.consumed_engine.clone(),
6812                package.reconciliation_replay.clone(),
6813            )
6814        };
6815
6816        let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
6817        for partition in &partitions {
6818            replay_disposition
6819                .register_partition(partition, false)
6820                .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
6821        }
6822        replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
6823        replay_disposition
6824            .register_consumed_members(consumed_engine.as_ref(), consumed_members.iter().copied());
6825        if !replay_disposition.consumed_engine_matches(&self.source_formula_token) {
6826            return Err(ExcelError::new(ExcelErrorKind::Value)
6827                .with_message("ResidualConsumedEngineMismatch"));
6828        }
6829        self.target_preparation_checkpoint(deadline, 1)?;
6830        let mut replay_records = if let Some(mut records) = reconciliation_replay {
6831            records.retain(|record| {
6832                let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
6833                else {
6834                    return true;
6835                };
6836                let coord = crate::engine::SourceCoord { row, col };
6837                let disposition = record.family.map_or_else(
6838                    || replay_disposition.ordinary_disposition(coord).0,
6839                    |family| replay_disposition.shared_disposition(family, coord),
6840                );
6841                !matches!(
6842                    disposition,
6843                    crate::engine::FormulaReplayCoordinateDisposition::Direct
6844                        | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
6845                )
6846            });
6847            records
6848        } else {
6849            replay
6850                .lock()
6851                .map_err(|_| {
6852                    ExcelError::new(ExcelErrorKind::Value)
6853                        .with_message("deferred formula spool lock poisoned")
6854                })?
6855                .replay_partitioned(&replay_disposition, &partitions)
6856                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
6857        };
6858        replay_records.sort_by_key(|record| record.source_order);
6859        for chunk in replay_records.chunks(256) {
6860            self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
6861        }
6862        if replay_records
6863            .windows(2)
6864            .any(|records| records[0].source_order == records[1].source_order)
6865        {
6866            return Err(ExcelError::new(ExcelErrorKind::Value)
6867                .with_message("duplicate deferred source-order proof"));
6868        }
6869
6870        let disposition = replay_disposition;
6871        let direct_families = 0usize;
6872        let direct_cells = 0u64;
6873        let direct_fragments = 0u64;
6874        let direct_complete_families = 0u64;
6875        let direct_complete_cells = 0u64;
6876        let direct_partition_families = 0u64;
6877        let direct_partition_cells = 0u64;
6878        let anchor_parses = 0u64;
6879        let anchor_asts = 0u64;
6880        let anchor_analyses = 0u64;
6881
6882        Ok(PreparedTargetSourcePackage {
6883            sheet: sheet.to_string(),
6884            sheet_id,
6885            lease,
6886            selected_points: None,
6887            complete_selections: Default::default(),
6888            deferred_shared: false,
6889            direct_domains: Vec::new(),
6890            source_report,
6891            replay_records,
6892            spool_replays: 1,
6893            disposition,
6894            legacy: Vec::new(),
6895            direct_families,
6896            direct_cells,
6897            direct_fragments,
6898            direct_complete_families,
6899            direct_complete_cells,
6900            direct_partition_families,
6901            direct_partition_cells,
6902            anchor_parses,
6903            anchor_asts,
6904            anchor_analyses,
6905        })
6906    }
6907
6908    fn fallback_planning_snapshot(
6909        &self,
6910        batch: &FormulaIngestBatch,
6911    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
6912        let mut requests = Vec::new();
6913        for formula in &batch.formulas {
6914            let ast = self
6915                .graph
6916                .data_store()
6917                .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
6918                .ok_or_else(|| {
6919                    ExcelError::new(ExcelErrorKind::Value)
6920                        .with_message("ordered fallback AST is unavailable")
6921                })?;
6922            Self::collect_planning_function_requests(&ast, &mut requests);
6923        }
6924        requests.sort();
6925        requests.dedup();
6926        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6927            &self.resolver,
6928            requests,
6929        )
6930        .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
6931    }
6932
6933    fn prepare_legacy_batch_fallback(
6934        &mut self,
6935        batch: FormulaIngestBatch,
6936        function_provider: &dyn crate::traits::FunctionProvider,
6937    ) -> Result<
6938        (
6939            crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
6940            u64,
6941        ),
6942        ExcelError,
6943    > {
6944        let formula_count = batch.formulas.len() as u64;
6945        let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6946        let mut planned = Vec::with_capacity(batch.formulas.len());
6947        for record in batch.formulas {
6948            let placement = CellRef::new(
6949                sheet_id,
6950                Coord::from_excel(record.row, record.col, true, true),
6951            );
6952            let ingested = self
6953                .graph
6954                .ingest_pipeline(function_provider)
6955                .enable_function_semantics()
6956                .ingest_formula(
6957                    FormulaAstInput::RawArena(record.ast_id),
6958                    placement,
6959                    record.formula_text,
6960                )
6961                .map_err(|error| {
6962                    ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
6963                })?;
6964            planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
6965        }
6966        let plan = self
6967            .graph
6968            .prepare_legacy_graph_plan(sheet_id, planned)
6969            .map_err(|error| {
6970                ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
6971            })?;
6972        Ok((plan, formula_count))
6973    }
6974
6975    fn publish_compressed_partial_report(
6976        &mut self,
6977        report: &FormulaIngestReport,
6978        direct_report: &FormulaIngestReport,
6979    ) {
6980        if direct_report.source_family_promoted == 0
6981            && direct_report.graph_formula_cells_materialized == 0
6982        {
6983            return;
6984        }
6985        let mut published = report.clone();
6986        published.accumulate(direct_report);
6987        self.record_formula_ingest_report(published);
6988    }
6989
6990    fn finish_compressed_formula_sources(
6991        &mut self,
6992        batches: Vec<(
6993            FormulaIngestBatch,
6994            crate::engine::FormulaCompressedSourceReport,
6995            crate::engine::FormulaCompressedPreparation,
6996        )>,
6997    ) -> Result<FormulaIngestReport, ExcelError> {
6998        self.observe_function_semantic_epoch()?;
6999        if batches.iter().any(|(_, _, preparation)| {
7000            !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
7001        }) {
7002            return Err(ExcelError::new(ExcelErrorKind::Value)
7003                .with_message("compressed source preparation belongs to another engine"));
7004        }
7005        if batches.iter().any(|(fallback, _, preparation)| {
7006            preparation.sheet_name.as_ref() != fallback.sheet_name
7007        }) {
7008            return Err(ExcelError::new(ExcelErrorKind::Value)
7009                .with_message("compressed source preparation sheet mismatch"));
7010        }
7011        let initial_guard = crate::function_registry::semantic_epoch_read_guard();
7012        let initial_provider_revision = self.resolver.planning_semantic_revision();
7013        let mut fallback_batches = Vec::with_capacity(batches.len());
7014        let mut stale_fallback_batches = Vec::new();
7015        let mut pending_preparations = Vec::new();
7016        for (mut fallback, mut source, mut preparation) in batches {
7017            for formula in fallback.formulas.drain(..) {
7018                let source_order = formula.source_order.ok_or_else(|| {
7019                    ExcelError::new(ExcelErrorKind::Value).with_message(
7020                        "compressed source supplied formulas without source-order proof",
7021                    )
7022                })?;
7023                let text = formula.formula_text.ok_or_else(|| {
7024                    ExcelError::new(ExcelErrorKind::Value).with_message(
7025                        "ordered compressed fallback formula has no exact source text",
7026                    )
7027                })?;
7028                preparation
7029                    .eager_replay
7030                    .push(crate::engine::DeferredReplayFormula {
7031                        source_order,
7032                        row: formula.row,
7033                        col: formula.col,
7034                        text: text.to_string(),
7035                        family: formula.source_family,
7036                        partition_owner: formula.partition_owner,
7037                    });
7038            }
7039            preparation
7040                .eager_replay
7041                .sort_by_key(|record| record.source_order);
7042            source.source_spool_replays = source
7043                .source_spool_replays
7044                .saturating_add(preparation.preparation_spool_replays);
7045            let stale_reason = preparation
7046                .function_semantics_used
7047                .then(|| {
7048                    if preparation.function_provider_revision != initial_provider_revision {
7049                        Some("FunctionProviderRevisionChanged")
7050                    } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
7051                    {
7052                        Some("FunctionSemanticEpochChanged")
7053                    } else {
7054                        None
7055                    }
7056                })
7057                .flatten();
7058            let stale_semantics = stale_reason.is_some();
7059            for reason in preparation.rejected.values() {
7060                *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
7061            }
7062            // No family is ever placed directly: every family replays.
7063            source.replay_families = source.families_seen;
7064            source.replay_cells = source.family_cells_seen;
7065            let compressed = crate::engine::FormulaCompressedSourceBatch::new(
7066                fallback.sheet_name.clone(),
7067                source,
7068            );
7069            if stale_semantics {
7070                stale_fallback_batches.push((fallback, compressed));
7071            } else {
7072                fallback_batches.push((fallback, compressed));
7073            }
7074            pending_preparations.push((preparation, stale_semantics));
7075        }
7076        drop(initial_guard);
7077
7078        // Build known fallback graphs first. Stale batches are forced through legacy ingest.
7079        let configured_mode = self.config.formula_plane_mode;
7080        self.config.formula_plane_mode = FormulaPlaneMode::Off;
7081        let stale_result =
7082            self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
7083        self.config.formula_plane_mode = configured_mode;
7084        let mut report = stale_result?;
7085        report.mode = configured_mode;
7086
7087        self.config.formula_plane_mode = FormulaPlaneMode::Off;
7088        let fallback_result =
7089            self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
7090        self.config.formula_plane_mode = configured_mode;
7091        match fallback_result {
7092            Ok(fallback_report) => report.accumulate(&fallback_report),
7093            Err(error) => {
7094                self.record_formula_ingest_report(report);
7095                return Err(error);
7096            }
7097        }
7098
7099        let mut direct_report =
7100            FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
7101        for (preparation, _) in &pending_preparations {
7102            direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
7103                preparation.clean_rejected_anchor_counts[0]
7104                    .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
7105            );
7106            direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
7107                preparation.clean_rejected_anchor_counts[1]
7108                    .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
7109            );
7110            direct_report.source_anchor_analyses =
7111                direct_report.source_anchor_analyses.saturating_add(
7112                    preparation.clean_rejected_anchor_counts[2]
7113                        .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
7114                );
7115        }
7116        loop {
7117            #[cfg(any(test, feature = "test-support"))]
7118            if let Some(hook) = self.before_prepared_span_commit_hook.take() {
7119                hook();
7120            }
7121            let commit_guard = crate::function_registry::semantic_epoch_read_guard();
7122            let commit_provider_revision = self.resolver.planning_semantic_revision();
7123            let mut newly_stale = Vec::new();
7124            let mut current = Vec::new();
7125            for pending in pending_preparations.drain(..) {
7126                let stale_reason = pending
7127                    .0
7128                    .function_semantics_used
7129                    .then(|| {
7130                        if pending.0.function_provider_revision != commit_provider_revision {
7131                            Some("FunctionProviderRevisionChanged")
7132                        } else if self
7133                            .prepared_function_semantics_changed(&pending.0, &commit_guard)
7134                        {
7135                            Some("FunctionSemanticEpochChanged")
7136                        } else {
7137                            None
7138                        }
7139                    })
7140                    .flatten();
7141                if let Some(reason) = stale_reason {
7142                    newly_stale.push((pending, reason));
7143                } else {
7144                    current.push(pending);
7145                }
7146            }
7147            if !newly_stale.is_empty() {
7148                drop(commit_guard);
7149                for ((mut preparation, was_initially_stale), reason) in newly_stale {
7150                    preparation
7151                        .eager_replay
7152                        .sort_by_key(|record| record.source_order);
7153                    direct_report.source_spool_replays =
7154                        direct_report.source_spool_replays.saturating_add(1);
7155                    if !was_initially_stale {
7156                        direct_report
7157                            .fallback_reasons
7158                            .entry(reason.to_string())
7159                            .or_default();
7160                    }
7161                    preparation.function_semantics_used = false;
7162                    if !preparation.eager_replay.is_empty() {
7163                        current.push((preparation, false));
7164                    }
7165                }
7166                pending_preparations = current;
7167                continue;
7168            }
7169            drop(commit_guard);
7170
7171            // Replay fallback families in source order.
7172            enum SourceProposal {
7173                KnownFallback {
7174                    source_order: crate::engine::SourceFormulaOrder,
7175                    records: Vec<crate::engine::DeferredReplayFormula>,
7176                },
7177            }
7178
7179            impl SourceProposal {
7180                fn source_order(&self) -> crate::engine::SourceFormulaOrder {
7181                    match self {
7182                        Self::KnownFallback { source_order, .. } => *source_order,
7183                    }
7184                }
7185            }
7186
7187            for (mut preparation, _) in current {
7188                let mut proposals = Vec::with_capacity(preparation.eager_replay.len());
7189                let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
7190                for record in preparation.eager_replay.drain(..) {
7191                    if let Some(owner) = record.partition_owner.or(record.family) {
7192                        family_fallbacks.entry(owner).or_default().push(record);
7193                    } else {
7194                        proposals.push(SourceProposal::KnownFallback {
7195                            source_order: record.source_order,
7196                            records: vec![record],
7197                        });
7198                    }
7199                }
7200                for (_, mut records) in family_fallbacks {
7201                    records.sort_by_key(|record| record.source_order);
7202                    if records
7203                        .windows(2)
7204                        .any(|window| window[0].source_order == window[1].source_order)
7205                    {
7206                        self.publish_compressed_partial_report(&report, &direct_report);
7207                        return Err(ExcelError::new(ExcelErrorKind::Value)
7208                            .with_message("duplicate exact-replay source-order proof"));
7209                    }
7210                    let Some(source_order) = records.first().map(|record| record.source_order)
7211                    else {
7212                        self.publish_compressed_partial_report(&report, &direct_report);
7213                        return Err(ExcelError::new(ExcelErrorKind::Value)
7214                            .with_message("empty exact-replay fallback family"));
7215                    };
7216                    proposals.push(SourceProposal::KnownFallback {
7217                        source_order,
7218                        records,
7219                    });
7220                }
7221                proposals.sort_by_key(SourceProposal::source_order);
7222                if proposals
7223                    .windows(2)
7224                    .any(|window| window[0].source_order() == window[1].source_order())
7225                {
7226                    self.publish_compressed_partial_report(&report, &direct_report);
7227                    return Err(ExcelError::new(ExcelErrorKind::Value)
7228                        .with_message("ambiguous compressed source-order proof"));
7229                }
7230
7231                for proposal in proposals {
7232                    match proposal {
7233                        SourceProposal::KnownFallback { records, .. } => {
7234                            let batch = match self.formula_batch_from_exact_replay(
7235                                preparation.sheet_name.as_ref(),
7236                                records,
7237                            ) {
7238                                Ok(batch) => batch,
7239                                Err(error) => {
7240                                    self.publish_compressed_partial_report(&report, &direct_report);
7241                                    return Err(error);
7242                                }
7243                            };
7244                            if batch.is_empty() {
7245                                continue;
7246                            }
7247                            let snapshot = match self.fallback_planning_snapshot(&batch) {
7248                                Ok(snapshot) => snapshot,
7249                                Err(error) => {
7250                                    self.publish_compressed_partial_report(&report, &direct_report);
7251                                    return Err(error);
7252                                }
7253                            };
7254                            let commit_guard =
7255                                crate::function_registry::semantic_epoch_read_guard();
7256                            let provider_revision_initial =
7257                                self.resolver.planning_semantic_revision();
7258                            if (commit_guard.epoch() != snapshot.epoch()
7259                                && snapshot.semantic_changes_affect_requests_since_guarded(
7260                                    &commit_guard,
7261                                    snapshot.epoch(),
7262                                ))
7263                                || snapshot.provider_revision().is_some_and(|revision| {
7264                                    Some(revision) != provider_revision_initial
7265                                })
7266                            {
7267                                drop(commit_guard);
7268                                self.publish_compressed_partial_report(&report, &direct_report);
7269                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7270                                    "ordered fallback planning snapshot became stale",
7271                                ));
7272                            }
7273                            let (plan, formula_count) = match self
7274                                .prepare_legacy_batch_fallback(batch, &snapshot)
7275                            {
7276                                Ok(plan) => plan,
7277                                Err(error) => {
7278                                    drop(commit_guard);
7279                                    self.publish_compressed_partial_report(&report, &direct_report);
7280                                    return Err(error);
7281                                }
7282                            };
7283                            let provider_revision_after =
7284                                self.resolver.planning_semantic_revision();
7285                            if provider_revision_after != provider_revision_initial {
7286                                drop(commit_guard);
7287                                self.publish_compressed_partial_report(&report, &direct_report);
7288                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7289                                    "function provider changed while preparing ordered fallback",
7290                                ));
7291                            }
7292                            let graph_vertices = plan.new_vertex_count();
7293                            let Some(graph_edges) = plan.planned_edge_count() else {
7294                                drop(commit_guard);
7295                                self.publish_compressed_partial_report(&report, &direct_report);
7296                                return Err(ExcelError::new(ExcelErrorKind::Value)
7297                                    .with_message("prepared ordered fallback size overflow"));
7298                            };
7299                            if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
7300                            {
7301                                drop(commit_guard);
7302                                self.publish_compressed_partial_report(&report, &direct_report);
7303                                return Err(error);
7304                            }
7305                            if let Err(error) =
7306                                self.graph.validate_prepared_legacy_graph_plan(&plan)
7307                            {
7308                                drop(commit_guard);
7309                                self.publish_compressed_partial_report(&report, &direct_report);
7310                                return Err(ExcelError::new(ExcelErrorKind::Value)
7311                                    .with_message(error.to_string()));
7312                            }
7313                            #[cfg(test)]
7314                            if let Some(hook) = self
7315                                .before_legacy_fallback_final_provider_sample_hook
7316                                .take()
7317                            {
7318                                hook();
7319                            }
7320                            let provider_revision_final =
7321                                self.resolver.planning_semantic_revision();
7322                            if provider_revision_final != provider_revision_initial {
7323                                drop(commit_guard);
7324                                self.publish_compressed_partial_report(&report, &direct_report);
7325                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7326                                    "function provider changed after ordered fallback validation",
7327                                ));
7328                            }
7329                            let graph_formulas =
7330                                self.graph.apply_prevalidated_legacy_graph_plan(plan);
7331                            direct_report.formula_cells_seen = direct_report
7332                                .formula_cells_seen
7333                                .saturating_add(formula_count);
7334                            direct_report.graph_formula_cells_materialized = direct_report
7335                                .graph_formula_cells_materialized
7336                                .saturating_add(graph_formulas as u64);
7337                            direct_report.graph_vertices_created = direct_report
7338                                .graph_vertices_created
7339                                .saturating_add(graph_vertices as u64);
7340                            direct_report.graph_edges_created = direct_report
7341                                .graph_edges_created
7342                                .saturating_add(graph_edges as u64);
7343                        }
7344                    }
7345                    #[cfg(test)]
7346                    if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
7347                        hook();
7348                    }
7349                }
7350            }
7351            break;
7352        }
7353        report.accumulate(&direct_report);
7354        self.record_formula_ingest_report(report.clone());
7355        Ok(report)
7356    }
7357    /// Ingest replayed per-cell formulas while preserving compressed source counters.
7358    pub(crate) fn ingest_compressed_formula_source_batches(
7359        &mut self,
7360        batches: Vec<(
7361            FormulaIngestBatch,
7362            crate::engine::FormulaCompressedSourceBatch,
7363        )>,
7364    ) -> Result<FormulaIngestReport, ExcelError> {
7365        self.ingest_compressed_formula_source_batches_inner(batches, true)
7366    }
7367
7368    fn ingest_compressed_formula_source_batches_inner(
7369        &mut self,
7370        batches: Vec<(
7371            FormulaIngestBatch,
7372            crate::engine::FormulaCompressedSourceBatch,
7373        )>,
7374        publish_report: bool,
7375    ) -> Result<FormulaIngestReport, ExcelError> {
7376        let mut source_counts = [0_u64; 11];
7377        let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
7378        let mut formula_batches = Vec::with_capacity(batches.len());
7379        let mut compressed_families = Vec::new();
7380        let mut partitioned_families = Vec::new();
7381        for (batch, compressed_batch) in batches {
7382            let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
7383            if sheet_name.as_ref() != batch.sheet_name {
7384                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7385                    "compressed formula source sheet does not match its replay batch",
7386                ));
7387            }
7388            compressed_families.push((batch.sheet_name.clone(), families));
7389            partitioned_families.push((batch.sheet_name.clone(), partitions));
7390            source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
7391            source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
7392            source_counts[2] =
7393                source_counts[2].saturating_add(compressed.source_shared_anchor_events);
7394            source_counts[3] =
7395                source_counts[3].saturating_add(compressed.source_shared_descendant_events);
7396            source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
7397            source_counts[5] =
7398                source_counts[5].saturating_add(compressed.source_formula_records_spooled);
7399            source_counts[6] =
7400                source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
7401            source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
7402            source_counts[8] =
7403                source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
7404            source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
7405            source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
7406            source_report.families_seen = source_report
7407                .families_seen
7408                .saturating_add(compressed.families_seen);
7409            source_report.family_cells_seen = source_report
7410                .family_cells_seen
7411                .saturating_add(compressed.family_cells_seen);
7412            source_report.source_clean_families = source_report
7413                .source_clean_families
7414                .saturating_add(compressed.source_clean_families);
7415            source_report.source_clean_cells = source_report
7416                .source_clean_cells
7417                .saturating_add(compressed.source_clean_cells);
7418            source_report.source_fragmentable_families = source_report
7419                .source_fragmentable_families
7420                .saturating_add(compressed.source_fragmentable_families);
7421            source_report.source_fragmentable_cells = source_report
7422                .source_fragmentable_cells
7423                .saturating_add(compressed.source_fragmentable_cells);
7424            source_report.source_fragment_count = source_report
7425                .source_fragment_count
7426                .saturating_add(compressed.source_fragment_count);
7427            source_report.source_isolated_fallback_cells = source_report
7428                .source_isolated_fallback_cells
7429                .saturating_add(compressed.source_isolated_fallback_cells);
7430            source_report.source_hole_exclusions = source_report
7431                .source_hole_exclusions
7432                .saturating_add(compressed.source_hole_exclusions);
7433            source_report.source_ordinary_exclusions = source_report
7434                .source_ordinary_exclusions
7435                .saturating_add(compressed.source_ordinary_exclusions);
7436            source_report.source_partition_failures = source_report
7437                .source_partition_failures
7438                .saturating_add(compressed.source_partition_failures);
7439            source_report.replay_families = source_report
7440                .replay_families
7441                .saturating_add(compressed.replay_families);
7442            source_report.replay_cells = source_report
7443                .replay_cells
7444                .saturating_add(compressed.replay_cells);
7445            source_report.forward_descendants = source_report
7446                .forward_descendants
7447                .saturating_add(compressed.forward_descendants);
7448            source_report.evidence_limit_fallbacks = source_report
7449                .evidence_limit_fallbacks
7450                .saturating_add(compressed.evidence_limit_fallbacks);
7451            source_report.evidence_peak_bytes = source_report
7452                .evidence_peak_bytes
7453                .max(compressed.evidence_peak_bytes);
7454            for (reason, count) in compressed.fallback_reasons {
7455                *source_report.fallback_reasons.entry(reason).or_default() += count;
7456            }
7457            formula_batches.push(batch);
7458        }
7459        self.ingest_formula_batches_inner(
7460            formula_batches,
7461            source_counts,
7462            Some(source_report),
7463            compressed_families,
7464            partitioned_families,
7465            publish_report,
7466        )
7467    }
7468
7469    pub fn ingest_formula_batches(
7470        &mut self,
7471        batches: Vec<FormulaIngestBatch>,
7472    ) -> Result<FormulaIngestReport, ExcelError> {
7473        let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
7474        let report = self.ingest_formula_batches_inner(
7475            batches,
7476            [0; 11],
7477            None,
7478            Vec::new(),
7479            Vec::new(),
7480            true,
7481        )?;
7482        if has_formulas {
7483            self.mark_topology_edited();
7484        }
7485        Ok(report)
7486    }
7487
7488    fn ingest_formula_batches_unpublished(
7489        &mut self,
7490        batches: Vec<FormulaIngestBatch>,
7491    ) -> Result<FormulaIngestReport, ExcelError> {
7492        self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
7493    }
7494
7495    fn ingest_formula_batches_inner(
7496        &mut self,
7497        batches: Vec<FormulaIngestBatch>,
7498        source_counts: [u64; 11],
7499        source_report: Option<crate::engine::FormulaCompressedSourceReport>,
7500        compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
7501        partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
7502        publish_report: bool,
7503    ) -> Result<FormulaIngestReport, ExcelError> {
7504        self.observe_function_semantic_epoch()?;
7505        let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7506        #[cfg(feature = "tracing")]
7507        let arena_nodes_before = self.graph.data_store().memory_usage().total_ast_nodes;
7508        #[cfg(feature = "tracing")]
7509        let route = if !partitioned_families.is_empty() {
7510            "partitioned"
7511        } else if !compressed_families.is_empty() {
7512            "compressed_source"
7513        } else if source_report.is_some() {
7514            "replay"
7515        } else {
7516            "ordinary"
7517        };
7518        let _ingest_span = crate::engine::trace::fz_span!(
7519            tracing::Level::INFO,
7520            "ingest",
7521            "ingest.batch",
7522            mode = ?FormulaPlaneMode::Off,
7523            route,
7524            formula_cells = formula_cells_seen
7525        );
7526        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
7527        let materialize_batches = batches;
7528        report.formula_cells_seen = formula_cells_seen;
7529        report.source_formula_events = source_counts[0];
7530        report.source_ordinary_events = source_counts[1];
7531        report.source_shared_anchor_events = source_counts[2];
7532        report.source_shared_descendant_events = source_counts[3];
7533        report.source_unknown_events = source_counts[4];
7534        report.source_formula_records_spooled = source_counts[5];
7535        report.source_spool_encoded_bytes = source_counts[6];
7536        report.source_spool_peak_memory_bytes = source_counts[7];
7537        report.source_spool_spilled_bytes = source_counts[8];
7538        report.source_spool_spill_files = source_counts[9];
7539        report.source_spool_replays = source_counts[10];
7540        if let Some(source) = source_report {
7541            report.source_families_seen = source.families_seen;
7542            report.source_family_cells_seen = source.family_cells_seen;
7543            report.source_family_shadow_eligible = source.source_clean_families;
7544            report.source_family_shadow_eligible_cells = source.source_clean_cells;
7545            report.source_partitioned_families_seen = source.source_fragmentable_families;
7546            report.source_partition_holes = source.source_hole_exclusions;
7547            report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
7548            report.source_partition_failures = source.source_partition_failures;
7549            report.source_partition_surviving_cells = source.source_fragmentable_cells;
7550            report.source_family_fallback = report
7551                .source_family_fallback
7552                .saturating_add(source.replay_families);
7553            report.source_family_fallback_cells = report
7554                .source_family_fallback_cells
7555                .saturating_add(source.replay_cells);
7556            report.source_forward_descendants = source.forward_descendants;
7557            report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
7558            report.source_evidence_peak_bytes = source.evidence_peak_bytes;
7559            for (reason, count) in source.fallback_reasons {
7560                let total = report.fallback_reasons.entry(reason).or_default();
7561                *total = total.saturating_add(count);
7562            }
7563        }
7564
7565        // A first load without graph admission plans in the builder, one
7566        // chunk at a time (a load error fails the load, so the builder's
7567        // partial application on a planning error is unobservable).
7568        if self.graph.first_load_assume_new()
7569            && !self.graph_admission_enabled()
7570            && !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
7571        {
7572            let mut builder =
7573                crate::engine::ingest_builder::BulkIngestBuilder::new(&mut self.graph);
7574            for batch in materialize_batches {
7575                if batch.is_empty() {
7576                    continue;
7577                }
7578                let sheet_id = builder
7579                    .add_sheet_checked(&batch.sheet_name)
7580                    .ok_or_else(|| {
7581                        ExcelError::new(ExcelErrorKind::Ref)
7582                            .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7583                    })?;
7584                builder.add_formula_refs(
7585                    sheet_id,
7586                    batch.formulas.into_iter().map(|record| {
7587                        (
7588                            record.row,
7589                            record.col,
7590                            crate::engine::graph::FormulaRef::of_ingested(
7591                                record.ast_id,
7592                                record.member_anchor,
7593                            ),
7594                        )
7595                    }),
7596                );
7597            }
7598            let summary = builder.finish_with_provider(&self.resolver)?;
7599            report.graph_formula_cells_materialized = summary.formulas as u64;
7600            report.graph_vertices_created = summary.vertices as u64;
7601            report.graph_edges_created = summary.edges as u64;
7602        } else if !materialize_batches.iter().all(FormulaIngestBatch::is_empty) {
7603            let mut prepared_by_sheet: BTreeMap<String, Vec<_>> = BTreeMap::new();
7604            for batch in materialize_batches {
7605                if batch.is_empty() {
7606                    continue;
7607                }
7608                let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
7609                    ExcelError::new(ExcelErrorKind::Ref)
7610                        .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7611                })?;
7612                let mut pipeline = self.ingest_pipeline();
7613                // Plan each record and keep only what the graph needs (the
7614                // pipeline's per-formula facts are dropped at once).
7615                let prepared = prepared_by_sheet.entry(batch.sheet_name).or_default();
7616                prepared.reserve(batch.formulas.len());
7617                for record in batch.formulas {
7618                    let placement = CellRef::new(
7619                        sheet_id,
7620                        Coord::from_excel(record.row, record.col, true, true),
7621                    );
7622                    let input = match record.member_anchor {
7623                        Some(anchor) => FormulaAstInput::Member {
7624                            template: record.ast_id,
7625                            anchor,
7626                        },
7627                        None => FormulaAstInput::RawArena(record.ast_id),
7628                    };
7629                    let formula = pipeline.ingest_formula(input, placement, None)?;
7630                    prepared.push((
7631                        record.row,
7632                        record.col,
7633                        crate::engine::graph::FormulaRef::of_ingested(
7634                            formula.ast_id,
7635                            formula.member_anchor,
7636                        ),
7637                        formula.dep_plan,
7638                    ));
7639                }
7640            }
7641            let admission_preflighted = self.graph_admission_enabled();
7642            if admission_preflighted {
7643                let mut preview = Vec::new();
7644                for (sheet_name, formulas) in &prepared_by_sheet {
7645                    let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
7646                        ExcelError::new(ExcelErrorKind::Ref)
7647                            .with_message(format!("unknown ingest sheet: {sheet_name}"))
7648                    })?;
7649                    preview.extend(
7650                        formulas
7651                            .iter()
7652                            .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
7653                    );
7654                }
7655                let admission = self.graph.preview_formula_mutations(&preview)?;
7656                self.preflight_graph_admission(admission)?;
7657            }
7658
7659            let mut builder = self.begin_bulk_ingest();
7660            if admission_preflighted {
7661                builder.mark_admission_preflighted();
7662            }
7663            for (sheet_name, formulas) in prepared_by_sheet {
7664                if formulas.is_empty() {
7665                    continue;
7666                }
7667                let sheet_id = builder.add_sheet(&sheet_name);
7668                builder.add_formula_plans(sheet_id, formulas);
7669            }
7670            let summary = builder.finish()?;
7671            report.graph_formula_cells_materialized = summary.formulas as u64;
7672            report.graph_vertices_created = summary.vertices as u64;
7673            report.graph_edges_created = summary.edges as u64;
7674        }
7675
7676        crate::engine::trace::fz_event!(
7677            tracing::Level::INFO,
7678            "ingest",
7679            "ingest.summary",
7680            candidate_cells = report.shadow_candidate_cells,
7681            accepted_span_cells = report.shadow_accepted_span_cells,
7682            fallback_cells = report.shadow_fallback_cells,
7683            spans_created = report.shadow_spans_created,
7684            templates_interned = report.shadow_templates_interned,
7685            graph_vertices_created = report.graph_vertices_created,
7686            graph_edges_created = report.graph_edges_created,
7687            arena_nodes_delta = self
7688                .graph
7689                .data_store()
7690                .memory_usage()
7691                .total_ast_nodes
7692                .saturating_sub(arena_nodes_before)
7693        );
7694        if publish_report {
7695            self.record_formula_ingest_report(report.clone());
7696        }
7697        Ok(report)
7698    }
7699
7700    fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
7701        let mut unique = Vec::new();
7702        for diagnostic in self.formula_parse_diagnostics.drain(start..) {
7703            let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
7704                prior.sheet == diagnostic.sheet
7705                    && prior.row == diagnostic.row
7706                    && prior.col == diagnostic.col
7707                    && prior.formula == diagnostic.formula
7708                    && prior.policy == diagnostic.policy
7709            });
7710            if !duplicate {
7711                unique.push(diagnostic);
7712            }
7713        }
7714        self.formula_parse_diagnostics.extend(unique);
7715    }
7716
7717    pub fn handle_formula_parse_error(
7718        &mut self,
7719        sheet: &str,
7720        row: u32,
7721        col: u32,
7722        formula: &str,
7723        message: String,
7724    ) -> Result<Option<ASTNode>, ExcelError> {
7725        let policy = self.config.formula_parse_policy;
7726
7727        if policy == FormulaParsePolicy::Strict {
7728            let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
7729            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7730                "Formula parse error at {sheet}!{col_a1}{row}: {message}"
7731            )));
7732        }
7733
7734        self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
7735            sheet: sheet.to_string(),
7736            row,
7737            col,
7738            formula: formula.to_string(),
7739            message: message.clone(),
7740            policy,
7741        });
7742
7743        match policy {
7744            FormulaParsePolicy::Strict => unreachable!(),
7745            FormulaParsePolicy::KeepCachedValue => Ok(None),
7746            FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
7747                ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
7748                None,
7749            ))),
7750            FormulaParsePolicy::CoerceToError => {
7751                let err = ExcelError::new(ExcelErrorKind::Error)
7752                    .with_message(format!("Malformed formula: {message}"));
7753                Ok(Some(ASTNode::new(
7754                    ASTNodeType::Literal(LiteralValue::Error(err)),
7755                    None,
7756                )))
7757            }
7758        }
7759    }
7760
7761    #[cfg(test)]
7762    fn target_preparation_fault(
7763        &mut self,
7764        seam: crate::engine::target_preparation::TargetPreparationFault,
7765    ) -> Result<(), ExcelError> {
7766        if self.target_preparation_fault_for_test == Some(seam) {
7767            self.target_preparation_fault_for_test = None;
7768            Err(ExcelError::new(ExcelErrorKind::Value)
7769                .with_message(format!("injected target preparation fault: {seam:?}")))
7770        } else {
7771            Ok(())
7772        }
7773    }
7774
7775    fn preparation_stale(
7776        reason: formualizer_common::PreparationStaleReason,
7777        message: impl Into<String>,
7778    ) -> ExcelError {
7779        ExcelError::new(ExcelErrorKind::Value)
7780            .with_message(message)
7781            .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
7782    }
7783
7784    fn preparation_revision_stale_reason(
7785        assumptions: &crate::engine::PreparationRevision,
7786        current: &crate::engine::PreparationRevision,
7787        planning_requests: &BTreeSet<(String, String, usize)>,
7788        staged_leases_match: bool,
7789    ) -> Option<formualizer_common::PreparationStaleReason> {
7790        if assumptions.graph != current.graph {
7791            Some(formualizer_common::PreparationStaleReason::Graph)
7792        } else if assumptions.staged != current.staged || !staged_leases_match {
7793            Some(formualizer_common::PreparationStaleReason::Staged)
7794        } else if assumptions.symbols != current.symbols {
7795            Some(formualizer_common::PreparationStaleReason::Symbols)
7796        } else if assumptions.provider != current.provider {
7797            Some(formualizer_common::PreparationStaleReason::Provider)
7798        } else if assumptions.semantic != current.semantic
7799            && crate::function_registry::semantic_changes_affect_requests_since(
7800                assumptions.semantic,
7801                planning_requests.iter().cloned(),
7802            )
7803        {
7804            Some(formualizer_common::PreparationStaleReason::Semantic)
7805        } else {
7806            None
7807        }
7808    }
7809
7810    fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
7811        crate::engine::PreparationRevision {
7812            graph: self.graph.topology_revision(),
7813            authority: 0,
7814            authority_indexes: 0,
7815            authority_indexed_plane: 0,
7816            staged: self.staged_formula_index.revision(),
7817            symbols: self.graph.symbol_revision(),
7818            semantic: crate::function_registry::semantic_epoch(),
7819            provider: self.resolver.planning_semantic_revision(),
7820        }
7821    }
7822
7823    fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
7824        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
7825        let provider = self.resolver.planning_semantic_revision();
7826        let semantic = registry_guard.epoch();
7827        PlanningRevisionSnapshot {
7828            engine_topology_epoch: self.topology_epoch,
7829            graph_topology_revision: self.graph.topology_revision(),
7830            staged: self.staged_formula_index.revision(),
7831            symbols: self.graph.symbol_revision(),
7832            semantic,
7833            provider,
7834            deterministic_mode: self.config.deterministic_mode.clone(),
7835            budgets: self.evaluation_resource_budgets.clone(),
7836        }
7837    }
7838
7839    fn recalc_plan_key(&self) -> RecalcPlanKey {
7840        RecalcPlanKey {
7841            engine_token: Arc::clone(&self.recalc_plan_token),
7842            revisions: self.planning_revision_snapshot(),
7843        }
7844    }
7845
7846    fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
7847        ExcelError::new(ExcelErrorKind::Value)
7848            .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
7849            .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
7850    }
7851
7852    fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
7853        use formualizer_common::PlanStaleReason;
7854
7855        if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
7856            return Err(Self::plan_stale(PlanStaleReason::Engine));
7857        }
7858
7859        let current = self.planning_revision_snapshot();
7860        let expected = &key.revisions;
7861        let stale = if expected.provider != current.provider {
7862            Some(PlanStaleReason::Provider)
7863        } else if expected.semantic != current.semantic {
7864            Some(PlanStaleReason::Semantic)
7865        } else if expected.budgets != current.budgets
7866            || expected.deterministic_mode != current.deterministic_mode
7867        {
7868            Some(PlanStaleReason::Budget)
7869        } else if expected.staged != current.staged {
7870            Some(PlanStaleReason::Staged)
7871        } else if expected.symbols != current.symbols {
7872            Some(PlanStaleReason::Symbols)
7873        } else if expected.graph_topology_revision != current.graph_topology_revision
7874            || expected.engine_topology_epoch != current.engine_topology_epoch
7875        {
7876            Some(PlanStaleReason::Graph)
7877        } else {
7878            None
7879        };
7880        stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
7881    }
7882
7883    fn target_preparation_checkpoint(
7884        &mut self,
7885        deadline: Option<std::time::Instant>,
7886        work: u64,
7887    ) -> Result<(), ExcelError> {
7888        if self
7889            .active_cancel_flag
7890            .as_ref()
7891            .is_some_and(|cancel| cancel.is_cancelled())
7892        {
7893            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
7894                .with_message("target graph preparation cancelled"));
7895        }
7896        if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
7897            return Err(crate::engine::ResourceLedgerError::Exhausted(
7898                formualizer_common::ResourceExhaustionDetail {
7899                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
7900                    limit: 0,
7901                    observed: 1,
7902                    request_id: self
7903                        .active_evaluation_resource_request
7904                        .as_ref()
7905                        .map(|stats| stats.request_id),
7906                },
7907            )
7908            .into_excel_error());
7909        }
7910        self.charge_bounded_work(work)
7911    }
7912
7913    fn opaque_reason_in_ast(
7914        &self,
7915        ast: &ASTNode,
7916        provider: &crate::function_registry::RegistryPlanningSnapshot,
7917    ) -> Option<crate::engine::OpaqueReason> {
7918        match &ast.node_type {
7919            ASTNodeType::Function { name, args } => {
7920                let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
7921                if canonical == "INDIRECT" {
7922                    return Some(crate::engine::OpaqueReason::RuntimeTextReference);
7923                }
7924                let Some(caps) = provider.function_capabilities("", name) else {
7925                    return Some(crate::engine::OpaqueReason::UnknownFunction);
7926                };
7927                if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
7928                    || caps.contains(FnCaps::RETURNS_REFERENCE)
7929                {
7930                    return Some(crate::engine::OpaqueReason::DynamicReference);
7931                }
7932                args.iter()
7933                    .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
7934            }
7935            ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
7936            ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
7937            ASTNodeType::BinaryOp { left, right, .. } => self
7938                .opaque_reason_in_ast(left, provider)
7939                .or_else(|| self.opaque_reason_in_ast(right, provider)),
7940            ASTNodeType::Array(rows) => rows
7941                .iter()
7942                .flat_map(|row| row.iter())
7943                .find_map(|item| self.opaque_reason_in_ast(item, provider)),
7944            ASTNodeType::Reference {
7945                reference:
7946                    ReferenceType::Cell {
7947                        sheet: Some(sheet), ..
7948                    }
7949                    | ReferenceType::Range {
7950                        sheet: Some(sheet), ..
7951                    },
7952                ..
7953            } if self.graph.sheet_id(sheet).is_none() => {
7954                Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
7955            }
7956            ASTNodeType::Reference {
7957                reference:
7958                    ReferenceType::External(_)
7959                    | ReferenceType::Cell3D { .. }
7960                    | ReferenceType::Range3D { .. },
7961                ..
7962            } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
7963            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
7964        }
7965    }
7966
7967    fn target_planning_snapshot(
7968        &mut self,
7969        ast: &ASTNode,
7970        planning_requests: &mut BTreeSet<(String, String, usize)>,
7971    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7972        #[cfg(test)]
7973        if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
7974            return Err(Self::preparation_stale(
7975                formualizer_common::PreparationStaleReason::Semantic,
7976                "injected target semantic preparation movement",
7977            ));
7978        }
7979        #[cfg(test)]
7980        if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
7981            hook();
7982        }
7983        let mut requests = Vec::new();
7984        Self::collect_planning_function_requests(ast, &mut requests);
7985        requests.sort();
7986        requests.dedup();
7987        planning_requests.extend(requests.iter().cloned());
7988        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7989            &self.resolver,
7990            requests,
7991        )
7992        .map_err(|error| {
7993            ExcelError::new(ExcelErrorKind::Value)
7994                .with_message(format!("target planning snapshot unavailable: {error:?}"))
7995        })
7996    }
7997
7998    fn target_planning_snapshot_stale_reason(
7999        snapshot: &crate::function_registry::RegistryPlanningSnapshot,
8000        assumptions: &crate::engine::PreparationRevision,
8001    ) -> Option<formualizer_common::PreparationStaleReason> {
8002        if snapshot
8003            .provider_revision()
8004            .is_some_and(|revision| Some(revision) != assumptions.provider)
8005        {
8006            Some(formualizer_common::PreparationStaleReason::Provider)
8007        } else if snapshot.epoch() != assumptions.semantic
8008            && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
8009        {
8010            Some(formualizer_common::PreparationStaleReason::Semantic)
8011        } else {
8012            None
8013        }
8014    }
8015
8016    fn widen_target_preparation(
8017        policy: crate::engine::OpaquePreparePolicy,
8018        scope: &mut crate::engine::PrepareScope,
8019        reasons: &mut Vec<crate::engine::OpaqueReason>,
8020        reason: crate::engine::OpaqueReason,
8021    ) -> Result<bool, ExcelError> {
8022        if policy == crate::engine::OpaquePreparePolicy::Error {
8023            return Err(ExcelError::new(ExcelErrorKind::NImpl)
8024                .with_message(format!("opaque target preparation semantics: {reason:?}")));
8025        }
8026        if !reasons.contains(&reason) {
8027            reasons.push(reason);
8028        }
8029        if !matches!(scope, crate::engine::PrepareScope::Workbook) {
8030            *scope = crate::engine::PrepareScope::Workbook;
8031            Ok(true)
8032        } else {
8033            Ok(false)
8034        }
8035    }
8036
8037    fn widen_target_preparation_to_sheet(
8038        policy: crate::engine::OpaquePreparePolicy,
8039        scope: &mut crate::engine::PrepareScope,
8040        reasons: &mut Vec<crate::engine::OpaqueReason>,
8041        reason: crate::engine::OpaqueReason,
8042        sheet: &str,
8043    ) -> Result<bool, ExcelError> {
8044        if policy == crate::engine::OpaquePreparePolicy::Error {
8045            return Err(ExcelError::new(ExcelErrorKind::NImpl)
8046                .with_message(format!("opaque target preparation semantics: {reason:?}")));
8047        }
8048        if !reasons.contains(&reason) {
8049            reasons.push(reason);
8050        }
8051        match scope {
8052            crate::engine::PrepareScope::Exact => {
8053                *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
8054                Ok(true)
8055            }
8056            crate::engine::PrepareScope::Sheets(sheets) => {
8057                if sheets.iter().any(|candidate| candidate == sheet) {
8058                    Ok(false)
8059                } else {
8060                    sheets.push(sheet.to_string());
8061                    sheets.sort();
8062                    Ok(true)
8063                }
8064            }
8065            crate::engine::PrepareScope::Workbook => Ok(false),
8066        }
8067    }
8068
8069    fn ast_has_proven_sheet_local_dynamic(
8070        ast: &ASTNode,
8071        provider: &crate::function_registry::RegistryPlanningSnapshot,
8072    ) -> bool {
8073        fn classify(
8074            ast: &ASTNode,
8075            provider: &crate::function_registry::RegistryPlanningSnapshot,
8076        ) -> (bool, bool) {
8077            match &ast.node_type {
8078                ASTNodeType::Function { name, args } => {
8079                    let Some(caps) = provider.function_capabilities("", name) else {
8080                        return (false, false);
8081                    };
8082                    let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
8083                        || caps.contains(FnCaps::RETURNS_REFERENCE);
8084                    let canonical = name.rsplit('.').next().unwrap_or(name);
8085                    if dynamic
8086                        && !canonical.eq_ignore_ascii_case("OFFSET")
8087                        && !canonical.eq_ignore_ascii_case("INDEX")
8088                    {
8089                        return (false, true);
8090                    }
8091                    let mut has_dynamic = dynamic;
8092                    for arg in args {
8093                        let (safe, child_dynamic) = classify(arg, provider);
8094                        if !safe {
8095                            return (false, has_dynamic || child_dynamic);
8096                        }
8097                        has_dynamic |= child_dynamic;
8098                    }
8099                    (true, has_dynamic)
8100                }
8101                ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
8102                ASTNodeType::BinaryOp { left, right, .. } => {
8103                    let (left_safe, left_dynamic) = classify(left, provider);
8104                    let (right_safe, right_dynamic) = classify(right, provider);
8105                    (left_safe && right_safe, left_dynamic || right_dynamic)
8106                }
8107                ASTNodeType::Array(rows) => {
8108                    let mut has_dynamic = false;
8109                    for item in rows.iter().flatten() {
8110                        let (safe, child_dynamic) = classify(item, provider);
8111                        if !safe {
8112                            return (false, has_dynamic || child_dynamic);
8113                        }
8114                        has_dynamic |= child_dynamic;
8115                    }
8116                    (true, has_dynamic)
8117                }
8118                ASTNodeType::Reference {
8119                    reference:
8120                        ReferenceType::Cell { sheet: None, .. }
8121                        | ReferenceType::Range { sheet: None, .. },
8122                    ..
8123                }
8124                | ASTNodeType::Literal(_)
8125                | ASTNodeType::Omitted => (true, false),
8126                ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
8127            }
8128        }
8129
8130        let (safe, dynamic) = classify(ast, provider);
8131        safe && dynamic
8132    }
8133
8134    fn table_selection_region(
8135        &self,
8136        entry: &crate::engine::graph::TableEntry,
8137        selection: &crate::engine::TableSelection,
8138    ) -> Result<PreparationRegion, ExcelError> {
8139        let mut start_row = entry.range.start.coord.row() + 1;
8140        let mut end_row = entry.range.end.coord.row() + 1;
8141        let mut start_col = entry.range.start.coord.col() + 1;
8142        let mut end_col = entry.range.end.coord.col() + 1;
8143        match selection {
8144            crate::engine::TableSelection::Whole => {}
8145            crate::engine::TableSelection::Headers => {
8146                if !entry.header_row {
8147                    return Err(ExcelError::new(ExcelErrorKind::Value)
8148                        .with_message(format!("table {} has no header row", entry.name)));
8149                }
8150                end_row = start_row;
8151            }
8152            crate::engine::TableSelection::Data => {
8153                if entry.header_row {
8154                    start_row = start_row.saturating_add(1);
8155                }
8156                if entry.totals_row {
8157                    end_row = end_row.saturating_sub(1);
8158                }
8159            }
8160            crate::engine::TableSelection::Totals => {
8161                if !entry.totals_row {
8162                    return Err(ExcelError::new(ExcelErrorKind::Value)
8163                        .with_message(format!("table {} has no totals row", entry.name)));
8164                }
8165                start_row = end_row;
8166            }
8167            crate::engine::TableSelection::Column(column) => {
8168                let index = entry.col_index(column).ok_or_else(|| {
8169                    ExcelError::new(ExcelErrorKind::Name)
8170                        .with_message(format!("unknown table column: {column}"))
8171                })?;
8172                start_col = start_col.saturating_add(index as u32);
8173                end_col = start_col;
8174            }
8175            crate::engine::TableSelection::Columns { start, end } => {
8176                let first = entry.col_index(start).ok_or_else(|| {
8177                    ExcelError::new(ExcelErrorKind::Name)
8178                        .with_message(format!("unknown table column: {start}"))
8179                })?;
8180                let last = entry.col_index(end).ok_or_else(|| {
8181                    ExcelError::new(ExcelErrorKind::Name)
8182                        .with_message(format!("unknown table column: {end}"))
8183                })?;
8184                if first > last {
8185                    return Err(ExcelError::new(ExcelErrorKind::Value)
8186                        .with_message("table column selection is reversed"));
8187                }
8188                start_col = start_col.saturating_add(first as u32);
8189                end_col = entry
8190                    .range
8191                    .start
8192                    .coord
8193                    .col()
8194                    .saturating_add(last as u32)
8195                    .saturating_add(1);
8196            }
8197        }
8198        if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
8199            start_row = entry.range.start.coord.row() + 1;
8200            end_row = start_row;
8201        }
8202        if start_row > end_row || start_col > end_col {
8203            return Err(
8204                ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
8205            );
8206        }
8207        Ok(PreparationRegion {
8208            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
8209            sheet_id: entry.sheet_id(),
8210            start_row,
8211            start_col,
8212            end_row,
8213            end_col,
8214        })
8215    }
8216
8217    #[cfg(test)]
8218    pub(crate) fn reset_target_root_dedup_probes_for_test() {
8219        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
8220    }
8221
8222    #[cfg(test)]
8223    pub(crate) fn target_root_dedup_probes_for_test() -> usize {
8224        TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
8225    }
8226
8227    pub(crate) fn resolve_target_producers(
8228        &mut self,
8229        targets: &[crate::engine::EvaluationTarget],
8230    ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
8231        use crate::engine::target_preparation::TargetProducer;
8232
8233        let request_id = self
8234            .active_evaluation_resource_request
8235            .as_ref()
8236            .map(|request| request.request_id);
8237        let mut roots = OrderedTargetProducers::with_capacity(targets.len())
8238            .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
8239        let resolve_region = |engine: &mut Self,
8240                              region: Region,
8241                              value_only: Option<CellRef>,
8242                              roots: &mut OrderedTargetProducers|
8243         -> Result<(), ExcelError> {
8244            let before = roots.len();
8245            for anchor in engine.graph.spill_anchors_in_region(
8246                region.sheet_id(),
8247                region.axis_ranges().0.query_bounds().0,
8248                region.axis_ranges().1.query_bounds().0,
8249                region.axis_ranges().0.query_bounds().1,
8250                region.axis_ranges().1.query_bounds().1,
8251            ) {
8252                roots
8253                    .push(TargetProducer::Legacy(anchor))
8254                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8255            }
8256            for vertex in engine.graph.vertices_in_region(
8257                region.sheet_id(),
8258                region.axis_ranges().0.query_bounds().0,
8259                region.axis_ranges().0.query_bounds().1,
8260                region.axis_ranges().1.query_bounds().0,
8261                region.axis_ranges().1.query_bounds().1,
8262            ) {
8263                let vertex = engine
8264                    .graph
8265                    .get_cell_ref(vertex)
8266                    .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
8267                    .unwrap_or(vertex);
8268                // `vertices_in_region` is a sheet-index query and a sheet index holds only
8269                // grid-addressed vertices, so a region can never yield a symbol: names,
8270                // tables and external sources have no position for a region to cover.
8271                // Symbol roots come from the by-name lookups below instead.
8272                if matches!(
8273                    engine.graph.get_vertex_kind(vertex),
8274                    VertexKind::FormulaScalar | VertexKind::FormulaArray
8275                ) {
8276                    roots
8277                        .push(TargetProducer::Legacy(vertex))
8278                        .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8279                }
8280            }
8281            if roots.len() == before
8282                && let Some(cell) = value_only
8283            {
8284                roots
8285                    .push(TargetProducer::ValueOnly(cell))
8286                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8287            }
8288            Ok(())
8289        };
8290
8291        for target in targets {
8292            match target {
8293                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8294                    if *row == 0 || *col == 0 {
8295                        return Err(ExcelError::new(ExcelErrorKind::Ref)
8296                            .with_message("target cell coordinates are one-based"));
8297                    }
8298                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8299                        ExcelError::new(ExcelErrorKind::Ref)
8300                            .with_message(format!("target sheet not found: {sheet}"))
8301                    })?;
8302                    let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
8303                    resolve_region(
8304                        self,
8305                        Region::point(sheet_id, *row - 1, *col - 1),
8306                        Some(cell),
8307                        &mut roots,
8308                    )?;
8309                }
8310                crate::engine::EvaluationTarget::Range(range) => {
8311                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8312                        ExcelError::new(ExcelErrorKind::Ref)
8313                            .with_message(format!("target sheet not found: {}", range.sheet))
8314                    })?;
8315                    resolve_region(
8316                        self,
8317                        Region::rect(
8318                            sheet_id,
8319                            range.start_row - 1,
8320                            range.end_row - 1,
8321                            range.start_col - 1,
8322                            range.end_col - 1,
8323                        ),
8324                        None,
8325                        &mut roots,
8326                    )?;
8327                }
8328                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8329                    let scope = self.name_query_scope(scope_sheet.as_deref())?;
8330                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
8331                        roots
8332                            .push(TargetProducer::Symbol(entry.vertex))
8333                            .map_err(|_| {
8334                                target_root_allocation_error(roots.len() + 1, request_id)
8335                            })?;
8336                    }
8337                }
8338                crate::engine::EvaluationTarget::Table { name, .. } => {
8339                    if let Some(entry) = self.graph.resolve_table_entry(name) {
8340                        roots
8341                            .push(TargetProducer::Symbol(entry.vertex))
8342                            .map_err(|_| {
8343                                target_root_allocation_error(roots.len() + 1, request_id)
8344                            })?;
8345                    }
8346                }
8347            }
8348        }
8349        Ok(roots.into_vec())
8350    }
8351
8352    /// Transactionally prepare the complete ordinary staged demand closure for typed targets.
8353    ///
8354    /// This method prepares graph topology only. It does not evaluate target values.
8355    pub fn prepare_graph_for_targets(
8356        &mut self,
8357        targets: &[crate::engine::EvaluationTarget],
8358        options: crate::engine::TargetEvalOptions<'_>,
8359    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8360        let previous_budgets = self.evaluation_resource_budgets.clone();
8361        let diagnostics_len = self.formula_parse_diagnostics.len();
8362        let previous_report = self.last_formula_ingest_report.clone();
8363        if let Some(budgets) = options.budgets {
8364            self.evaluation_resource_budgets = budgets.clone();
8365        }
8366        let previous_graph_budget_override = self
8367            .graph
8368            .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
8369        // Hoist the call's cancellation onto the engine for its duration, so the
8370        // preparation checkpoints observe it. This is a standalone entry point, so
8371        // the previous value is restored rather than cleared.
8372        let previous_cancel = self.active_cancel_flag.take();
8373        self.active_cancel_flag = options.cancel.clone();
8374        let result = self.observe_evaluation_resource_request(
8375            EvaluationRequestKind::TargetPreparation,
8376            |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
8377        );
8378        self.active_cancel_flag = previous_cancel;
8379        self.graph
8380            .set_admission_budget_override(previous_graph_budget_override);
8381        self.evaluation_resource_budgets = previous_budgets;
8382        if result.is_err() {
8383            self.formula_parse_diagnostics.truncate(diagnostics_len);
8384            self.last_formula_ingest_report = previous_report;
8385        }
8386        result
8387    }
8388
8389    fn prepare_graph_for_targets_unobserved(
8390        &mut self,
8391        targets: &[crate::engine::EvaluationTarget],
8392        options: &crate::engine::TargetEvalOptions<'_>,
8393    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8394        let scratch_checkpoint = self
8395            .active_resource_ledger
8396            .as_ref()
8397            .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
8398        let result = self.prepare_graph_for_targets_transaction(targets, options);
8399        let release = self
8400            .active_resource_ledger
8401            .as_mut()
8402            .map_or(Ok(()), |ledger| {
8403                ledger.release_scratch_to(scratch_checkpoint)
8404            });
8405        match (result, release) {
8406            (result, Ok(())) => result,
8407            (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
8408        }
8409    }
8410
8411    fn prepare_graph_for_targets_transaction(
8412        &mut self,
8413        targets: &[crate::engine::EvaluationTarget],
8414        options: &crate::engine::TargetEvalOptions<'_>,
8415    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8416        use crate::engine::{
8417            OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
8418            TableSelection,
8419        };
8420
8421        self.target_preparation_checkpoint(options.deadline, 0)?;
8422        self.observe_function_semantic_epoch()?;
8423        let assumptions = self.preparation_revisions();
8424        let ledger_at_start = self
8425            .active_resource_ledger
8426            .as_ref()
8427            .map(|ledger| ledger.snapshot());
8428        let diagnostics_len = self.formula_parse_diagnostics.len();
8429        let report_len = self.last_formula_ingest_report.clone();
8430        let request_id = options.request_id.or_else(|| {
8431            self.active_evaluation_resource_request
8432                .as_ref()
8433                .map(|stats| stats.request_id)
8434        });
8435
8436        let mut scope = PrepareScope::Exact;
8437        let mut reasons = Vec::new();
8438        let mut regions = VecDeque::new();
8439        let mut deferred_shared_regions = VecDeque::new();
8440        let mut normalized = Vec::with_capacity(targets.len());
8441        let mut symbol_vertices = VecDeque::new();
8442        for target in targets {
8443            self.target_preparation_checkpoint(options.deadline, 1)?;
8444            match target {
8445                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8446                    if *row == 0 || *col == 0 {
8447                        return Err(ExcelError::new(ExcelErrorKind::Ref)
8448                            .with_message("target cell coordinates are one-based"));
8449                    }
8450                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8451                        ExcelError::new(ExcelErrorKind::Ref)
8452                            .with_message(format!("target sheet not found: {sheet}"))
8453                    })?;
8454                    regions.push_back(PreparationRegion {
8455                        sheet: sheet.clone(),
8456                        sheet_id,
8457                        start_row: *row,
8458                        start_col: *col,
8459                        end_row: *row,
8460                        end_col: *col,
8461                    });
8462                    normalized.push(target.clone());
8463                }
8464                crate::engine::EvaluationTarget::Range(range) => {
8465                    if range.start_row == 0
8466                        || range.start_col == 0
8467                        || range.end_row < range.start_row
8468                        || range.end_col < range.start_col
8469                    {
8470                        return Err(ExcelError::new(ExcelErrorKind::Ref)
8471                            .with_message("invalid target range"));
8472                    }
8473                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8474                        ExcelError::new(ExcelErrorKind::Ref)
8475                            .with_message(format!("target sheet not found: {}", range.sheet))
8476                    })?;
8477                    regions.push_back(PreparationRegion {
8478                        sheet: range.sheet.clone(),
8479                        sheet_id,
8480                        start_row: range.start_row,
8481                        start_col: range.start_col,
8482                        end_row: range.end_row,
8483                        end_col: range.end_col,
8484                    });
8485                    normalized.push(target.clone());
8486                }
8487                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8488                    let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
8489                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
8490                        symbol_vertices.push_back(entry.vertex);
8491                    } else {
8492                        Self::widen_target_preparation(
8493                            options.opaque_policy,
8494                            &mut scope,
8495                            &mut reasons,
8496                            OpaqueReason::UnresolvedName,
8497                        )?;
8498                    }
8499                    normalized.push(target.clone());
8500                }
8501                crate::engine::EvaluationTarget::Table { name, selection } => {
8502                    if let Some(entry) = self.graph.resolve_table_entry(name) {
8503                        let region = self.table_selection_region(entry, selection)?;
8504                        symbol_vertices.push_back(entry.vertex);
8505                        regions.push_back(region);
8506                    } else {
8507                        Self::widen_target_preparation(
8508                            options.opaque_policy,
8509                            &mut scope,
8510                            &mut reasons,
8511                            OpaqueReason::UnresolvedTable,
8512                        )?;
8513                    }
8514                    normalized.push(target.clone());
8515                }
8516            }
8517        }
8518
8519        let mut visited_regions = FxHashSet::default();
8520        let mut visited_vertices = FxHashSet::default();
8521        let mut selected = FxHashSet::default();
8522        let mut prepared = Vec::new();
8523        let mut pending_diagnostics = Vec::new();
8524        let mut planning_requests = BTreeSet::new();
8525        let mut selected_cells = Vec::new();
8526        let mut workbook_seeded = false;
8527        let mut sheet_scope_seeded = BTreeSet::new();
8528        let mut indexed_query_sheets = FxHashSet::default();
8529        let mut discovery_scratch_reserved = 0u64;
8530        let mut package_encountered = false;
8531        let mut selected_package_sheets = FxHashSet::default();
8532        let mut selected_package_points: BTreeMap<String, BTreeSet<(u32, u32)>> = BTreeMap::new();
8533        let mut prepared_packages: Vec<PreparedTargetSourcePackage> = Vec::new();
8534        let authoritative_with_ordinary = false;
8535        let has_unknown_package_sheet = self
8536            .staged_formula_index
8537            .package_sheets()
8538            .any(|sheet| self.graph.sheet_id(sheet).is_none());
8539
8540        loop {
8541            if let PrepareScope::Sheets(sheets) = &scope {
8542                for sheet in sheets.clone() {
8543                    if !sheet_scope_seeded.insert(sheet.clone()) {
8544                        continue;
8545                    }
8546                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8547                        package_encountered = true;
8548                        Self::widen_target_preparation(
8549                            options.opaque_policy,
8550                            &mut scope,
8551                            &mut reasons,
8552                            OpaqueReason::UnsupportedSourceSemantics,
8553                        )?;
8554                        continue;
8555                    };
8556                    for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
8557                        self.target_preparation_checkpoint(options.deadline, 1)?;
8558                        regions.push_back(PreparationRegion {
8559                            sheet: sheet.clone(),
8560                            sheet_id,
8561                            start_row: lease.row,
8562                            start_col: lease.col,
8563                            end_row: lease.row,
8564                            end_col: lease.col,
8565                        });
8566                    }
8567                    if self
8568                        .staged_formula_index
8569                        .package_lease_for_sheet(&sheet)
8570                        .is_some()
8571                    {
8572                        regions.push_back(PreparationRegion {
8573                            sheet: sheet.clone(),
8574                            sheet_id,
8575                            start_row: 1,
8576                            start_col: 1,
8577                            end_row: self.workbook_load_limits.max_sheet_rows,
8578                            end_col: self.workbook_load_limits.max_sheet_cols,
8579                        });
8580                    }
8581                }
8582            }
8583            if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
8584                workbook_seeded = true;
8585                for (sheet, lease) in self.staged_formula_index.all_leases() {
8586                    self.target_preparation_checkpoint(options.deadline, 1)?;
8587                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8588                        package_encountered = true;
8589                        Self::widen_target_preparation(
8590                            options.opaque_policy,
8591                            &mut scope,
8592                            &mut reasons,
8593                            OpaqueReason::UnsupportedSourceSemantics,
8594                        )?;
8595                        continue;
8596                    };
8597                    regions.push_back(PreparationRegion {
8598                        sheet,
8599                        sheet_id,
8600                        start_row: lease.row,
8601                        start_col: lease.col,
8602                        end_row: lease.row,
8603                        end_col: lease.col,
8604                    });
8605                }
8606                let package_sheets = self
8607                    .staged_formula_index
8608                    .package_sheets()
8609                    .map(str::to_string)
8610                    .collect::<Vec<_>>();
8611                for sheet in package_sheets {
8612                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8613                        package_encountered = true;
8614                        Self::widen_target_preparation(
8615                            options.opaque_policy,
8616                            &mut scope,
8617                            &mut reasons,
8618                            OpaqueReason::UnsupportedSourceSemantics,
8619                        )?;
8620                        continue;
8621                    };
8622                    regions.push_back(PreparationRegion {
8623                        sheet,
8624                        sheet_id,
8625                        start_row: 1,
8626                        start_col: 1,
8627                        end_row: self.workbook_load_limits.max_sheet_rows,
8628                        end_col: self.workbook_load_limits.max_sheet_cols,
8629                    });
8630                }
8631            }
8632
8633            // Finish known ordinary/name dependency discovery before expanding
8634            // partial shared demands. A queued SUM may complete those families.
8635            let allow_partial_shared = regions.is_empty() && symbol_vertices.is_empty();
8636            let next_region = if allow_partial_shared {
8637                deferred_shared_regions.pop_front()
8638            } else {
8639                regions.pop_front()
8640            };
8641            let Some(region) = next_region else {
8642                if let Some(vertex) = symbol_vertices.pop_front() {
8643                    self.target_preparation_checkpoint(options.deadline, 1)?;
8644                    if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
8645                        continue;
8646                    }
8647                    let vertex_is_dynamic = self.graph.is_dynamic(vertex);
8648                    if let Some(ast) = self.graph.get_formula(vertex) {
8649                        let snapshot =
8650                            self.target_planning_snapshot(&ast, &mut planning_requests)?;
8651                        if let Some(reason) =
8652                            Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
8653                        {
8654                            return Err(Self::preparation_stale(
8655                                reason,
8656                                "target planning snapshot became stale during discovery",
8657                            ));
8658                        }
8659                        let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
8660                        if let Some(reason) =
8661                            opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
8662                        {
8663                            if reason == OpaqueReason::DynamicReference
8664                                && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
8665                            {
8666                                let sheet = self.graph.get_vertex_sheet_id(vertex);
8667                                let sheet = self.graph.sheet_name(sheet).to_string();
8668                                Self::widen_target_preparation_to_sheet(
8669                                    options.opaque_policy,
8670                                    &mut scope,
8671                                    &mut reasons,
8672                                    reason,
8673                                    &sheet,
8674                                )?;
8675                            } else {
8676                                Self::widen_target_preparation(
8677                                    options.opaque_policy,
8678                                    &mut scope,
8679                                    &mut reasons,
8680                                    reason,
8681                                )?;
8682                            }
8683                        }
8684                    } else if vertex_is_dynamic {
8685                        Self::widen_target_preparation(
8686                            options.opaque_policy,
8687                            &mut scope,
8688                            &mut reasons,
8689                            OpaqueReason::DynamicReference,
8690                        )?;
8691                    }
8692                    if let Some(anchor) = self
8693                        .graph
8694                        .get_cell_ref(vertex)
8695                        .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
8696                    {
8697                        symbol_vertices.push_back(anchor);
8698                    }
8699                    if let Some(cell) = self.graph.get_cell_ref(vertex) {
8700                        let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
8701                        regions.push_back(PreparationRegion {
8702                            sheet,
8703                            sheet_id: cell.sheet_id,
8704                            start_row: cell.coord.row() + 1,
8705                            start_col: cell.coord.col() + 1,
8706                            end_row: cell.coord.row() + 1,
8707                            end_col: cell.coord.col() + 1,
8708                        });
8709                    }
8710                    // The formula's direct precedents, from the authority:
8711                    // cells and ranges become regions, symbol rows (names,
8712                    // tables, sources) their vertices.
8713                    match self.graph.authority_vertex_precedents(vertex) {
8714                        Some(precedents) => {
8715                            for (sheet_id, rect) in precedents {
8716                                self.target_preparation_checkpoint(options.deadline, 1)?;
8717                                if sheet_id == crate::engine::authority::geom::SYMBOL_SHEET {
8718                                    for slot in rect.r0..=rect.r1 {
8719                                        if let Some(symbol) =
8720                                            self.graph.authority_host().symbols().vertex(slot)
8721                                        {
8722                                            symbol_vertices.push_back(symbol);
8723                                        }
8724                                    }
8725                                    continue;
8726                                }
8727                                let sheet = self.graph.sheet_name(sheet_id).to_string();
8728                                regions.push_back(PreparationRegion {
8729                                    sheet,
8730                                    sheet_id,
8731                                    start_row: rect.r0 + 1,
8732                                    start_col: rect.c0 + 1,
8733                                    end_row: (rect.r1 + 1)
8734                                        .min(self.workbook_load_limits.max_sheet_rows),
8735                                    end_col: (rect.c1 + 1)
8736                                        .min(self.workbook_load_limits.max_sheet_cols),
8737                                });
8738                            }
8739                        }
8740                        // The authority cannot answer (failed host): widen.
8741                        None => {
8742                            if Self::widen_target_preparation(
8743                                options.opaque_policy,
8744                                &mut scope,
8745                                &mut reasons,
8746                                OpaqueReason::UnresolvedCrossSheetBinding,
8747                            )? {
8748                                continue;
8749                            }
8750                        }
8751                    }
8752                    if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
8753                        match &name.definition {
8754                            NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
8755                                sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
8756                                sheet_id: cell.sheet_id,
8757                                start_row: cell.coord.row() + 1,
8758                                start_col: cell.coord.col() + 1,
8759                                end_row: cell.coord.row() + 1,
8760                                end_col: cell.coord.col() + 1,
8761                            }),
8762                            NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
8763                                sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
8764                                sheet_id: range.start.sheet_id,
8765                                start_row: range.start.coord.row() + 1,
8766                                start_col: range.start.coord.col() + 1,
8767                                end_row: range.end.coord.row() + 1,
8768                                end_col: range.end.coord.col() + 1,
8769                            }),
8770                            NamedDefinition::Formula {
8771                                ast,
8772                                dependencies,
8773                                range_deps,
8774                            } => {
8775                                let snapshot =
8776                                    self.target_planning_snapshot(ast, &mut planning_requests)?;
8777                                if let Some(reason) = Self::target_planning_snapshot_stale_reason(
8778                                    &snapshot,
8779                                    &assumptions,
8780                                ) {
8781                                    return Err(Self::preparation_stale(
8782                                        reason,
8783                                        "target planning snapshot became stale during discovery",
8784                                    ));
8785                                }
8786                                if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
8787                                    Self::widen_target_preparation(
8788                                        options.opaque_policy,
8789                                        &mut scope,
8790                                        &mut reasons,
8791                                        reason,
8792                                    )?;
8793                                }
8794                                for dependency in dependencies {
8795                                    self.target_preparation_checkpoint(options.deadline, 1)?;
8796                                    symbol_vertices.push_back(*dependency);
8797                                }
8798                                for range in range_deps {
8799                                    self.target_preparation_checkpoint(options.deadline, 1)?;
8800                                    // `Current` is the sheet this name's formula
8801                                    // was interpreted on, which is the sheet its
8802                                    // vertex is placed on -- the same derivation
8803                                    // the formula-vertex arm above uses. It is
8804                                    // never the workbook default sheet, and an
8805                                    // unresolvable `Name` widens instead of
8806                                    // silently landing on some other sheet.
8807                                    let context_sheet = self.graph.get_vertex_sheet_id(vertex);
8808                                    let Ok(sheet_id) =
8809                                        self.resolve_sheet_locator(&range.sheet, context_sheet)
8810                                    else {
8811                                        if Self::widen_target_preparation(
8812                                            options.opaque_policy,
8813                                            &mut scope,
8814                                            &mut reasons,
8815                                            OpaqueReason::UnresolvedCrossSheetBinding,
8816                                        )? {
8817                                            break;
8818                                        }
8819                                        continue;
8820                                    };
8821                                    regions.push_back(PreparationRegion {
8822                                        sheet: self.graph.sheet_name(sheet_id).to_string(),
8823                                        sheet_id,
8824                                        start_row: range
8825                                            .start_row
8826                                            .map_or(1, |bound| bound.index + 1),
8827                                        start_col: range
8828                                            .start_col
8829                                            .map_or(1, |bound| bound.index + 1),
8830                                        end_row: range.end_row.map_or(
8831                                            self.workbook_load_limits.max_sheet_rows,
8832                                            |bound| bound.index + 1,
8833                                        ),
8834                                        end_col: range.end_col.map_or(
8835                                            self.workbook_load_limits.max_sheet_cols,
8836                                            |bound| bound.index + 1,
8837                                        ),
8838                                    });
8839                                }
8840                            }
8841                            NamedDefinition::Literal(_) => {}
8842                        }
8843                    }
8844                    if let Some(table) = self.graph.table_by_vertex(vertex) {
8845                        regions
8846                            .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
8847                    }
8848                    continue;
8849                }
8850                break;
8851            };
8852
8853            self.target_preparation_checkpoint(options.deadline, 1)?;
8854            if !visited_regions.insert(region.clone()) && !allow_partial_shared {
8855                continue;
8856            }
8857            if let PrepareScope::Sheets(sheets) = &mut scope
8858                && !sheets.iter().any(|sheet| sheet == &region.sheet)
8859            {
8860                sheets.push(region.sheet.clone());
8861                sheets.sort();
8862            }
8863            let package_match = self.staged_formula_index.package_for_region(
8864                &region.sheet,
8865                region.start_row,
8866                region.start_col,
8867                region.end_row,
8868                region.end_col,
8869            );
8870            let package_lease = match package_match {
8871                Some(Ok(lease)) => Some(lease),
8872                Some(Err(()))
8873                    if region.start_row == 1
8874                        && region.start_col == 1
8875                        && region.end_row == self.workbook_load_limits.max_sheet_rows
8876                        && region.end_col == self.workbook_load_limits.max_sheet_cols =>
8877                {
8878                    self.staged_formula_index
8879                        .package_lease_for_sheet(&region.sheet)
8880                }
8881                Some(Err(())) => {
8882                    Self::widen_target_preparation(
8883                        options.opaque_policy,
8884                        &mut scope,
8885                        &mut reasons,
8886                        OpaqueReason::DeferredSourcePackage,
8887                    )?;
8888                    None
8889                }
8890                None => None,
8891            };
8892            let compatibility_before_package_replay = package_lease.is_some()
8893                && (authoritative_with_ordinary || has_unknown_package_sheet);
8894            let package_lease = if compatibility_before_package_replay {
8895                package_encountered = true;
8896                Self::widen_target_preparation(
8897                    options.opaque_policy,
8898                    &mut scope,
8899                    &mut reasons,
8900                    OpaqueReason::UnsupportedSourceSemantics,
8901                )?;
8902                None
8903            } else {
8904                package_lease
8905            };
8906            if let Some(package_lease) = package_lease
8907                && !selected_package_sheets.contains(&region.sheet)
8908            {
8909                self.target_preparation_checkpoint(options.deadline, 1)?;
8910                let mut points = self.staged_formula_index.package_points_in_region(
8911                    &region.sheet,
8912                    region.start_row,
8913                    region.start_col,
8914                    region.end_row,
8915                    region.end_col,
8916                );
8917                if let Some(selected) = selected_package_points.get(&region.sheet) {
8918                    points.retain(|point| !selected.contains(point));
8919                }
8920
8921                if let Some(selected) = selected_package_points.get(&region.sheet) {
8922                    points.retain(|point| !selected.contains(point));
8923                }
8924                let permit_partial = allow_partial_shared
8925                    || (points.len() == 1
8926                        && regions.is_empty()
8927                        && symbol_vertices.is_empty()
8928                        && deferred_shared_regions.is_empty());
8929                let partial = self.prepare_target_exact_source_selection(
8930                    &region.sheet,
8931                    package_lease,
8932                    points,
8933                    selected_package_points
8934                        .get(&region.sheet)
8935                        .unwrap_or(&BTreeSet::new()),
8936                    permit_partial,
8937                    options.deadline,
8938                    &mut discovery_scratch_reserved,
8939                )?;
8940                let mut package = if let Some(mut package) = partial {
8941                    if package.deferred_shared {
8942                        deferred_shared_regions.push_back(region.clone());
8943                    }
8944                    // A later range can complete a family touched earlier in this
8945                    // request. Retire only that family's earlier legacy proposals;
8946                    // nothing has been published and its dependencies remain demanded.
8947                    if !package.direct_domains.is_empty() {
8948                        for prior in prepared_packages
8949                            .iter_mut()
8950                            .filter(|prior| prior.sheet == region.sheet)
8951                        {
8952                            prior
8953                                .replay_records
8954                                .retain(|record| !package.direct_contains(record.row, record.col));
8955                            prior
8956                                .legacy
8957                                .retain(|(row, col, _, _)| !package.direct_contains(*row, *col));
8958                            if let Some(points) = prior.selected_points.as_mut() {
8959                                points.retain(|&(row, col)| !package.direct_contains(row, col));
8960                            }
8961                        }
8962                    }
8963                    if selected_package_points.contains_key(&region.sheet) {
8964                        package.source_report = Default::default();
8965                    }
8966                    let points = package.selected_points.as_ref().unwrap();
8967                    if !points.is_empty() {
8968                        selected_package_points
8969                            .entry(region.sheet.clone())
8970                            .or_default()
8971                            .extend(points.iter().copied());
8972                    }
8973                    package
8974                } else {
8975                    selected_package_sheets.insert(region.sheet.clone());
8976                    self.prepare_target_source_package(
8977                        &region.sheet,
8978                        package_lease,
8979                        options.deadline,
8980                    )?
8981                };
8982                if package
8983                    .selected_points
8984                    .as_ref()
8985                    .is_none_or(|points| !points.is_empty())
8986                {
8987                    let mut final_fallback = BTreeMap::new();
8988                    for record in package.fallback_records() {
8989                        final_fallback.insert((record.row, record.col), record.clone());
8990                    }
8991                    let batch = self.formula_batch_from_exact_replay(
8992                        &region.sheet,
8993                        final_fallback.into_values(),
8994                    )?;
8995                    for record in batch.formulas {
8996                        self.target_preparation_checkpoint(options.deadline, 1)?;
8997                        let ast = self
8998                            .graph
8999                            .data_store()
9000                            .retrieve_ast(record.ast_id, self.graph.sheet_reg())
9001                            .ok_or_else(|| {
9002                                ExcelError::new(ExcelErrorKind::Value)
9003                                    .with_message("target fallback AST is unavailable")
9004                            })?;
9005                        let snapshot =
9006                            self.target_planning_snapshot(&ast, &mut planning_requests)?;
9007                        if let Some(reason) =
9008                            Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9009                        {
9010                            return Err(Self::preparation_stale(
9011                                reason,
9012                                "target fallback planning snapshot became stale during discovery",
9013                            ));
9014                        }
9015                        let proven_sheet_local_dynamic =
9016                            Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9017                        if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9018                            if reason == OpaqueReason::DynamicReference
9019                                && proven_sheet_local_dynamic
9020                            {
9021                                Self::widen_target_preparation_to_sheet(
9022                                    options.opaque_policy,
9023                                    &mut scope,
9024                                    &mut reasons,
9025                                    reason,
9026                                    &region.sheet,
9027                                )?;
9028                            } else {
9029                                Self::widen_target_preparation(
9030                                    options.opaque_policy,
9031                                    &mut scope,
9032                                    &mut reasons,
9033                                    reason,
9034                                )?;
9035                            }
9036                        }
9037                        let placement = CellRef::new(
9038                            package.sheet_id,
9039                            Coord::from_excel(record.row, record.col, true, true),
9040                        );
9041                        let ingested = self
9042                            .graph
9043                            .ingest_pipeline(&snapshot)
9044                            .enable_function_semantics()
9045                            .ingest_formula(
9046                                FormulaAstInput::RawArena(record.ast_id),
9047                                placement,
9048                                record.formula_text,
9049                            )?;
9050                        if ingested.dep_plan.dynamic {
9051                            if proven_sheet_local_dynamic {
9052                                Self::widen_target_preparation_to_sheet(
9053                                    options.opaque_policy,
9054                                    &mut scope,
9055                                    &mut reasons,
9056                                    OpaqueReason::DynamicReference,
9057                                    &region.sheet,
9058                                )?;
9059                            } else {
9060                                Self::widen_target_preparation(
9061                                    options.opaque_policy,
9062                                    &mut scope,
9063                                    &mut reasons,
9064                                    OpaqueReason::DynamicReference,
9065                                )?;
9066                            }
9067                        }
9068                        for dep in &ingested.dep_plan.direct_cell_deps {
9069                            self.target_preparation_checkpoint(options.deadline, 1)?;
9070                            regions.push_back(PreparationRegion {
9071                                sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9072                                sheet_id: dep.sheet_id,
9073                                start_row: dep.coord.row().saturating_add(1),
9074                                start_col: dep.coord.col().saturating_add(1),
9075                                end_row: dep.coord.row().saturating_add(1),
9076                                end_col: dep.coord.col().saturating_add(1),
9077                            });
9078                        }
9079                        for range in &ingested.dep_plan.range_deps {
9080                            self.target_preparation_checkpoint(options.deadline, 1)?;
9081                            // `Current` is the sheet the staged package's formula
9082                            // lives on.
9083                            let Ok(dependency_sheet) =
9084                                self.resolve_sheet_locator(&range.sheet, package.sheet_id)
9085                            else {
9086                                Self::widen_target_preparation(
9087                                    options.opaque_policy,
9088                                    &mut scope,
9089                                    &mut reasons,
9090                                    OpaqueReason::UnresolvedCrossSheetBinding,
9091                                )?;
9092                                continue;
9093                            };
9094                            regions.push_back(PreparationRegion {
9095                                sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9096                                sheet_id: dependency_sheet,
9097                                start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9098                                start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9099                                end_row: range
9100                                    .end_row
9101                                    .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9102                                        bound.index + 1
9103                                    }),
9104                                end_col: range
9105                                    .end_col
9106                                    .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9107                                        bound.index + 1
9108                                    }),
9109                            });
9110                        }
9111                        for name in ingested
9112                            .dep_plan
9113                            .resolved_named_refs
9114                            .iter()
9115                            .chain(&ingested.dep_plan.named_refs)
9116                        {
9117                            self.target_preparation_checkpoint(options.deadline, 1)?;
9118                            if let Some(entry) =
9119                                self.graph.resolve_name_entry(name, package.sheet_id)
9120                            {
9121                                symbol_vertices.push_back(entry.vertex);
9122                            } else if self.graph.resolve_source_scalar_entry(name).is_none()
9123                                && self.graph.resolve_source_table_entry(name).is_none()
9124                            {
9125                                Self::widen_target_preparation(
9126                                    options.opaque_policy,
9127                                    &mut scope,
9128                                    &mut reasons,
9129                                    OpaqueReason::UnresolvedName,
9130                                )?;
9131                            }
9132                        }
9133                        for table in &ingested.dep_plan.table_refs {
9134                            self.target_preparation_checkpoint(options.deadline, 1)?;
9135                            if let Some(entry) = self.graph.resolve_table_entry(table) {
9136                                symbol_vertices.push_back(entry.vertex);
9137                            } else if self.graph.resolve_source_table_entry(table).is_none() {
9138                                Self::widen_target_preparation(
9139                                    options.opaque_policy,
9140                                    &mut scope,
9141                                    &mut reasons,
9142                                    OpaqueReason::UnresolvedTable,
9143                                )?;
9144                            }
9145                        }
9146                        package.legacy.push((
9147                            record.row,
9148                            record.col,
9149                            ingested.ast_id,
9150                            ingested.dep_plan,
9151                        ));
9152                    }
9153                    prepared_packages.push(package);
9154                }
9155            }
9156            let leases = self.staged_formula_index.leases_in_region(
9157                &region.sheet,
9158                region.start_row,
9159                region.start_col,
9160                region.end_row,
9161                region.end_col,
9162            );
9163            for lease in leases {
9164                self.target_preparation_checkpoint(options.deadline, 1)?;
9165                let sheet_id = self.graph.sheet_id(&region.sheet).ok_or_else(|| {
9166                    ExcelError::new(ExcelErrorKind::Ref)
9167                        .with_message(format!("staged formula sheet not found: {}", region.sheet))
9168                })?;
9169                let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
9170                if !selected.insert(key) {
9171                    continue;
9172                }
9173                let text = self
9174                    .staged_formulas
9175                    .get(&region.sheet)
9176                    .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
9177                    .ok_or_else(|| {
9178                        ExcelError::new(ExcelErrorKind::Value)
9179                            .with_message("staged formula index is stale")
9180                    })?
9181                    .to_string();
9182                let formula = if text.starts_with('=') {
9183                    text.clone()
9184                } else {
9185                    format!("={text}")
9186                };
9187                self.target_preparation_checkpoint(options.deadline, 1)?;
9188                let ast = match formualizer_parse::parser::parse(&formula) {
9189                    Ok(ast) => ast,
9190                    Err(error) => {
9191                        if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
9192                            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9193                                format!(
9194                                    "Formula parse error at {}!{}{}: {error}",
9195                                    region.sheet,
9196                                    col_letters_from_1based(lease.col)
9197                                        .unwrap_or_else(|_| "?".to_string()),
9198                                    lease.row
9199                                ),
9200                            ));
9201                        }
9202                        pending_diagnostics.push(FormulaParseDiagnostic {
9203                            sheet: region.sheet.clone(),
9204                            row: lease.row,
9205                            col: lease.col,
9206                            formula: formula.clone(),
9207                            message: error.to_string(),
9208                            policy: self.config.formula_parse_policy,
9209                        });
9210                        match self.config.formula_parse_policy {
9211                            FormulaParsePolicy::KeepCachedValue => {
9212                                selected_cells.push(
9213                                    formualizer_common::RangeAddress::new(
9214                                        region.sheet.clone(),
9215                                        lease.row,
9216                                        lease.col,
9217                                        lease.row,
9218                                        lease.col,
9219                                    )
9220                                    .expect("selected staged coordinates are valid"),
9221                                );
9222                                prepared.push(PreparedOrdinaryStagedFormula {
9223                                    sheet: region.sheet.clone(),
9224                                    sheet_id,
9225                                    lease,
9226                                    ast_id: None,
9227                                    plan: None,
9228                                });
9229                                continue;
9230                            }
9231                            FormulaParsePolicy::AsText => ASTNode::new(
9232                                ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
9233                                None,
9234                            ),
9235                            FormulaParsePolicy::CoerceToError => ASTNode::new(
9236                                ASTNodeType::Literal(LiteralValue::Error(
9237                                    ExcelError::new(ExcelErrorKind::Error)
9238                                        .with_message(format!("Malformed formula: {error}")),
9239                                )),
9240                                None,
9241                            ),
9242                            FormulaParsePolicy::Strict => unreachable!(),
9243                        }
9244                    }
9245                };
9246                self.target_preparation_checkpoint(options.deadline, 1)?;
9247                let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
9248                self.target_preparation_checkpoint(options.deadline, 1)?;
9249                if let Some(reason) =
9250                    Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9251                {
9252                    return Err(Self::preparation_stale(
9253                        reason,
9254                        "target planning snapshot became stale during discovery",
9255                    ));
9256                }
9257                if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9258                    if reason == OpaqueReason::DynamicReference
9259                        && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
9260                    {
9261                        Self::widen_target_preparation_to_sheet(
9262                            options.opaque_policy,
9263                            &mut scope,
9264                            &mut reasons,
9265                            reason,
9266                            &region.sheet,
9267                        )?;
9268                    } else {
9269                        Self::widen_target_preparation(
9270                            options.opaque_policy,
9271                            &mut scope,
9272                            &mut reasons,
9273                            reason,
9274                        )?;
9275                    }
9276                }
9277                let proven_sheet_local_dynamic =
9278                    Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9279                let placement = CellRef::new(
9280                    sheet_id,
9281                    Coord::from_excel(lease.row, lease.col, true, true),
9282                );
9283                let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
9284                    FormulaAstInput::Tree(ast),
9285                    placement,
9286                    Some(Arc::from(formula)),
9287                )?;
9288                self.target_preparation_checkpoint(options.deadline, 1)?;
9289                if ingested.dep_plan.dynamic {
9290                    if proven_sheet_local_dynamic {
9291                        Self::widen_target_preparation_to_sheet(
9292                            options.opaque_policy,
9293                            &mut scope,
9294                            &mut reasons,
9295                            OpaqueReason::DynamicReference,
9296                            &region.sheet,
9297                        )?;
9298                    } else {
9299                        Self::widen_target_preparation(
9300                            options.opaque_policy,
9301                            &mut scope,
9302                            &mut reasons,
9303                            OpaqueReason::DynamicReference,
9304                        )?;
9305                    }
9306                }
9307                for dep in &ingested.dep_plan.direct_cell_deps {
9308                    self.target_preparation_checkpoint(options.deadline, 1)?;
9309                    regions.push_back(PreparationRegion {
9310                        sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9311                        sheet_id: dep.sheet_id,
9312                        start_row: dep.coord.row() + 1,
9313                        start_col: dep.coord.col() + 1,
9314                        end_row: dep.coord.row() + 1,
9315                        end_col: dep.coord.col() + 1,
9316                    });
9317                }
9318                for range in &ingested.dep_plan.range_deps {
9319                    self.target_preparation_checkpoint(options.deadline, 1)?;
9320                    // `Current` is the sheet the staged formula lives on.
9321                    let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
9322                    else {
9323                        Self::widen_target_preparation(
9324                            options.opaque_policy,
9325                            &mut scope,
9326                            &mut reasons,
9327                            OpaqueReason::UnresolvedCrossSheetBinding,
9328                        )?;
9329                        continue;
9330                    };
9331                    regions.push_back(PreparationRegion {
9332                        sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9333                        sheet_id: dependency_sheet,
9334                        start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9335                        start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9336                        end_row: range
9337                            .end_row
9338                            .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9339                                bound.index + 1
9340                            }),
9341                        end_col: range
9342                            .end_col
9343                            .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9344                                bound.index + 1
9345                            }),
9346                    });
9347                }
9348                for name in ingested
9349                    .dep_plan
9350                    .resolved_named_refs
9351                    .iter()
9352                    .chain(&ingested.dep_plan.named_refs)
9353                {
9354                    self.target_preparation_checkpoint(options.deadline, 1)?;
9355                    if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
9356                        symbol_vertices.push_back(entry.vertex);
9357                    } else if self.graph.resolve_source_scalar_entry(name).is_none()
9358                        && self.graph.resolve_source_table_entry(name).is_none()
9359                    {
9360                        Self::widen_target_preparation(
9361                            options.opaque_policy,
9362                            &mut scope,
9363                            &mut reasons,
9364                            OpaqueReason::UnresolvedName,
9365                        )?;
9366                    }
9367                }
9368                for table in &ingested.dep_plan.table_refs {
9369                    self.target_preparation_checkpoint(options.deadline, 1)?;
9370                    if let Some(entry) = self.graph.resolve_table_entry(table) {
9371                        symbol_vertices.push_back(entry.vertex);
9372                    } else if self.graph.resolve_source_table_entry(table).is_none() {
9373                        Self::widen_target_preparation(
9374                            options.opaque_policy,
9375                            &mut scope,
9376                            &mut reasons,
9377                            OpaqueReason::UnresolvedTable,
9378                        )?;
9379                    }
9380                }
9381                selected_cells.push(
9382                    formualizer_common::RangeAddress::new(
9383                        region.sheet.clone(),
9384                        lease.row,
9385                        lease.col,
9386                        lease.row,
9387                        lease.col,
9388                    )
9389                    .expect("selected staged coordinates are valid"),
9390                );
9391                prepared.push(PreparedOrdinaryStagedFormula {
9392                    sheet: region.sheet.clone(),
9393                    sheet_id,
9394                    lease,
9395                    ast_id: Some(ingested.ast_id),
9396                    plan: Some(ingested.dep_plan),
9397                });
9398            }
9399
9400            if indexed_query_sheets.insert(region.sheet_id) {
9401                self.graph.prepare_sheet_index_for_query(region.sheet_id);
9402                let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
9403                    .saturating_mul(32);
9404                self.reserve_graph_source_scratch(bytes)?;
9405                discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
9406            }
9407            let spill_anchors = self.graph.spill_anchors_in_region(
9408                region.sheet_id,
9409                region.start_row - 1,
9410                region.start_col - 1,
9411                region.end_row - 1,
9412                region.end_col - 1,
9413            );
9414            for anchor in spill_anchors {
9415                self.target_preparation_checkpoint(options.deadline, 1)?;
9416                symbol_vertices.push_back(anchor);
9417            }
9418            let vertices = self.graph.vertices_in_region(
9419                region.sheet_id,
9420                region.start_row - 1,
9421                region.end_row - 1,
9422                region.start_col - 1,
9423                region.end_col - 1,
9424            );
9425            for vertex in vertices {
9426                self.target_preparation_checkpoint(options.deadline, 1)?;
9427                symbol_vertices.push_back(vertex);
9428            }
9429        }
9430
9431        #[cfg(test)]
9432        self.target_preparation_fault(
9433            crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
9434        )?;
9435
9436        if package_encountered {
9437            Self::widen_target_preparation(
9438                options.opaque_policy,
9439                &mut scope,
9440                &mut reasons,
9441                OpaqueReason::UnsupportedSourceSemantics,
9442            )?;
9443            self.target_preparation_checkpoint(options.deadline, 0)?;
9444            let selected_count = self.staged_formula_count();
9445            let selected_packages = self
9446                .staged_formulas
9447                .values()
9448                .filter_map(|staged| staged.deferred_package.as_ref())
9449                .map(|package| package.families.len() + package.partitioned_families.len())
9450                .sum();
9451            self.formula_parse_diagnostics.truncate(diagnostics_len);
9452            self.last_formula_ingest_report = report_len;
9453            #[cfg(test)]
9454            if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9455                hook();
9456            }
9457            self.target_preparation_checkpoint(options.deadline, 0)?;
9458            #[cfg(test)]
9459            self.target_preparation_fault(
9460                crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9461            )?;
9462            let current_revisions = self.preparation_revisions();
9463            if let Some(reason) = Self::preparation_revision_stale_reason(
9464                &assumptions,
9465                &current_revisions,
9466                &planning_requests,
9467                true,
9468            ) {
9469                return Err(Self::preparation_stale(
9470                    reason,
9471                    "target compatibility preparation plan is stale",
9472                ));
9473            }
9474            #[cfg(test)]
9475            self.target_preparation_fault(
9476                crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9477            )?;
9478            let commit_work_before = self
9479                .active_resource_ledger
9480                .as_ref()
9481                .map_or(0, |ledger| ledger.snapshot().work_charged);
9482            let commit_started = crate::instant::FzInstant::now();
9483            self.build_graph_all_unobserved()?;
9484            let commit_window = commit_started.elapsed();
9485            let ledger_after = self
9486                .active_resource_ledger
9487                .as_ref()
9488                .map(|ledger| ledger.snapshot());
9489            let actual_commit_work = ledger_after
9490                .map_or(0, |snapshot| snapshot.work_charged)
9491                .saturating_sub(commit_work_before);
9492            let observed_scratch_bytes = ledger_after
9493                .map_or(0, |snapshot| snapshot.scratch_peak)
9494                .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
9495            let revisions = assumptions.clone();
9496            let report = PreparedTargetGraphReport {
9497                request_id: request_id.unwrap_or_default(),
9498                requested_targets: targets.len(),
9499                normalized_regions: visited_regions.len(),
9500                normalized_target_list: normalized,
9501                selected_staged_cells: selected_count,
9502                selected_source_families: selected_packages,
9503                retained_staged_cells: self.staged_formula_count(),
9504                selected_cells,
9505                retained_cells: Vec::new(),
9506                widened_scope: PrepareScope::Workbook,
9507                widening_reasons: reasons,
9508                revisions,
9509                commit_window,
9510                estimated_scratch_bytes: discovery_scratch_reserved
9511                    .saturating_add((selected_count as u64).saturating_mul(256)),
9512                observed_scratch_bytes,
9513                estimated_commit_work: selected_count as u64,
9514                actual_commit_work,
9515                outcome: PreparationOutcome::CompatibilityPrepared,
9516            };
9517            self.observe_target_preparation_report(&report);
9518            return Ok(report);
9519        }
9520
9521        prepared.sort_by_key(|formula| formula.lease.insertion_order);
9522        for package in &prepared_packages {
9523            if let Some(points) = &package.selected_points {
9524                selected_cells.extend(points.iter().filter_map(|&(row, col)| {
9525                    formualizer_common::RangeAddress::new(&package.sheet, row, col, row, col).ok()
9526                }));
9527                continue;
9528            }
9529            selected_cells.extend(package.replay_records.iter().filter_map(|record| {
9530                formualizer_common::RangeAddress::new(
9531                    package.sheet.clone(),
9532                    record.row,
9533                    record.col,
9534                    record.row,
9535                    record.col,
9536                )
9537                .ok()
9538            }));
9539        }
9540        let (legacy_graph, planned_formula_count) =
9541            self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
9542        let new_vertices = legacy_graph.new_vertex_count();
9543        let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
9544            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9545        })?;
9546        let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
9547            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9548        })?;
9549        let current = self.graph.baseline_stats();
9550        let final_vertices = current
9551            .graph_vertex_count
9552            .checked_add(new_vertices)
9553            .ok_or_else(|| {
9554                crate::engine::ResourceLedgerError::Exhausted(
9555                    formualizer_common::ResourceExhaustionDetail {
9556                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9557                        limit: u64::MAX,
9558                        observed: u64::MAX,
9559                        request_id,
9560                    },
9561                )
9562                .into_excel_error()
9563            })?;
9564        let final_edges = current
9565            .graph_edge_count
9566            .checked_sub(removed_edges)
9567            .and_then(|count| count.checked_add(new_edges))
9568            .ok_or_else(|| {
9569                crate::engine::ResourceLedgerError::Exhausted(
9570                    formualizer_common::ResourceExhaustionDetail {
9571                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9572                        limit: u64::MAX,
9573                        observed: u64::MAX,
9574                        request_id,
9575                    },
9576                )
9577                .into_excel_error()
9578            })?;
9579        #[cfg(test)]
9580        self.target_preparation_fault(
9581            crate::engine::target_preparation::TargetPreparationFault::Admission,
9582        )?;
9583        let resource = |reason, limit: u64, observed: u64| {
9584            crate::engine::ResourceLedgerError::Exhausted(
9585                formualizer_common::ResourceExhaustionDetail {
9586                    reason,
9587                    limit,
9588                    observed,
9589                    request_id,
9590                },
9591            )
9592            .into_excel_error()
9593        };
9594        let admission = crate::engine::resource_ledger::GraphAdmission {
9595            final_vertices,
9596            final_edges,
9597            materialization_cells: planned_formula_count as u64,
9598            added_vertices: new_vertices,
9599            added_edges: new_edges,
9600        };
9601        let materialized_bytes = admission
9602            .materialized_graph_bytes()
9603            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
9604        if let Err(error) = self.preflight_graph_admission(admission) {
9605            if let formualizer_common::ExcelErrorExtra::Resource { detail } = error.extra() {
9606                self.observe_target_admission_failure(detail.reason);
9607            }
9608            return Err(error);
9609        }
9610        let selected_package_records = prepared_packages
9611            .iter()
9612            .map(|package| package.replay_records.len() as u64)
9613            .sum::<u64>();
9614        let planned_working_bytes = (prepared.len() as u64)
9615            .saturating_add(selected_package_records)
9616            .saturating_mul(256)
9617            .saturating_add((visited_regions.len() as u64).saturating_mul(128))
9618            .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
9619            .saturating_add((selected.len() as u64).saturating_mul(96))
9620            .saturating_add(materialized_bytes);
9621        let residual_scratch = selected_package_points
9622            .iter()
9623            .map(|(sheet, points)| {
9624                let source = self
9625                    .staged_formulas
9626                    .get(sheet)
9627                    .unwrap()
9628                    .deferred_package
9629                    .as_ref()
9630                    .unwrap();
9631                (source.families.len() as u64)
9632                    .saturating_mul(256)
9633                    .saturating_add(
9634                        source
9635                            .partitioned_families
9636                            .iter()
9637                            .map(|family| {
9638                                256u64
9639                                    .saturating_add(family.fragments.len() as u64 * 32)
9640                                    .saturating_add(family.legacy_members.len() as u64 * 32)
9641                            })
9642                            .sum::<u64>(),
9643                    )
9644                    // Direct complete domains need coordinate suppression, not
9645                    // a per-member residual split/proof or legacy AST reserve.
9646                    .saturating_add(points.len() as u64 * 32)
9647                    .saturating_add(
9648                        prepared_packages
9649                            .iter()
9650                            .filter(|package| &package.sheet == sheet)
9651                            .map(|package| package.replay_records.len() as u64 * 480)
9652                            .sum::<u64>(),
9653                    )
9654            })
9655            .sum::<u64>();
9656        let scratch_bytes = discovery_scratch_reserved
9657            .saturating_add(planned_working_bytes)
9658            .saturating_add(residual_scratch);
9659        let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
9660        if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
9661            self.observe_target_admission_failure(
9662                formualizer_common::ResourceExhaustionReason::ScratchMemory,
9663            );
9664            return Err(error);
9665        }
9666
9667        let mut residual_sources = BTreeMap::new();
9668        let mut residual_owners = BTreeMap::new();
9669        for (sheet, points) in &selected_package_points {
9670            let selected: BTreeMap<_, BTreeSet<_>> = prepared_packages
9671                .iter()
9672                .filter(|package| &package.sheet == sheet)
9673                .flat_map(|package| &package.replay_records)
9674                .filter_map(|record| {
9675                    record.partition_owner.or(record.family).map(|owner| {
9676                        (
9677                            owner,
9678                            crate::engine::SourceCoord {
9679                                row: record.row - 1,
9680                                col: record.col - 1,
9681                            },
9682                        )
9683                    })
9684                })
9685                .fold(BTreeMap::new(), |mut map, (owner, coord)| {
9686                    map.entry(owner).or_default().insert(coord);
9687                    map
9688                });
9689            let metadata_work = self
9690                .staged_formulas
9691                .get(sheet)
9692                .unwrap()
9693                .deferred_package
9694                .as_ref()
9695                .map_or(0, |source| {
9696                    source.families.len() as u64
9697                        + source
9698                            .partitioned_families
9699                            .iter()
9700                            .map(|family| {
9701                                1 + family.fragments.len() as u64 * family.fragments.len() as u64
9702                                    + family.legacy_members.len() as u64
9703                            })
9704                            .sum::<u64>()
9705                });
9706            let split_work = selected
9707                .values()
9708                .map(|points| points.len() as u64 * 128)
9709                .sum::<u64>();
9710            self.target_preparation_checkpoint(
9711                options.deadline,
9712                metadata_work
9713                    .saturating_add(split_work)
9714                    .saturating_add(points.len() as u64),
9715            )?;
9716            let source = self
9717                .staged_formulas
9718                .get(sheet)
9719                .unwrap()
9720                .deferred_package
9721                .as_ref()
9722                .unwrap();
9723            let complete: BTreeSet<_> = prepared_packages
9724                .iter()
9725                .filter(|package| &package.sheet == sheet)
9726                .flat_map(|package| package.complete_selections.iter().copied())
9727                .collect();
9728            let residual = source
9729                .residual_sources(&selected, &complete, &self.workbook_load_limits)
9730                .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
9731            residual_owners.insert(
9732                sheet.clone(),
9733                residual
9734                    .1
9735                    .iter()
9736                    .map(|family| family.source_id)
9737                    .collect::<BTreeSet<_>>(),
9738            );
9739            residual_sources.insert(sheet.clone(), residual);
9740        }
9741
9742        // Reserve residual suppression before the revision-validated commit window.
9743        // Hash-set insertion and point-index removal below cannot allocate.
9744        for (sheet, points) in &selected_package_points {
9745            let package = self
9746                .staged_formulas
9747                .get_mut(sheet)
9748                .unwrap()
9749                .deferred_package
9750                .as_mut()
9751                .unwrap();
9752            let owner_count = prepared_packages
9753                .iter()
9754                .filter(|p| &p.sheet == sheet)
9755                .flat_map(|p| &p.replay_records)
9756                .filter_map(|record| record.partition_owner.or(record.family))
9757                .filter(|owner| residual_owners[sheet].contains(owner))
9758                .count();
9759            package
9760                .consumed_members
9761                .try_reserve(owner_count)
9762                .map_err(|_| {
9763                    resource(
9764                        formualizer_common::ResourceExhaustionReason::ScratchMemory,
9765                        0,
9766                        owner_count as u64 * 24,
9767                    )
9768                })?;
9769            package.suppressed.try_reserve(points.len()).map_err(|_| {
9770                resource(
9771                    formualizer_common::ResourceExhaustionReason::ScratchMemory,
9772                    0,
9773                    points.len() as u64 * 16,
9774                )
9775            })?;
9776        }
9777
9778        let estimated_commit_duration = std::time::Duration::from_nanos(
9779            (new_vertices as u64)
9780                .saturating_add(new_edges as u64)
9781                .saturating_add(prepared.len() as u64)
9782                .saturating_add(selected_package_records)
9783                .max(1)
9784                .saturating_mul(100),
9785        );
9786        if options.deadline.is_some_and(|deadline| {
9787            std::time::Instant::now()
9788                .checked_add(estimated_commit_duration)
9789                .is_none_or(|finish| finish > deadline)
9790        }) {
9791            self.observe_target_admission_failure(
9792                formualizer_common::ResourceExhaustionReason::Deadline,
9793            );
9794            return Err(resource(
9795                formualizer_common::ResourceExhaustionReason::Deadline,
9796                0,
9797                1,
9798            ));
9799        }
9800        #[cfg(test)]
9801        if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9802            hook();
9803        }
9804        self.target_preparation_checkpoint(options.deadline, 0)?;
9805        #[cfg(test)]
9806        self.target_preparation_fault(
9807            crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9808        )?;
9809        let current_revisions = self.preparation_revisions();
9810        let staged_leases_match = prepared.iter().all(|formula| {
9811            self.staged_formula_index
9812                .lease_matches(&formula.sheet, formula.lease)
9813        }) && prepared_packages.iter().all(|package| {
9814            self.staged_formula_index
9815                .package_lease_matches(&package.sheet, package.lease)
9816        });
9817        let stale_reason = Self::preparation_revision_stale_reason(
9818            &assumptions,
9819            &current_revisions,
9820            &planning_requests,
9821            staged_leases_match,
9822        );
9823        if let Some(reason) = stale_reason {
9824            return Err(Self::preparation_stale(
9825                reason,
9826                "target graph preparation plan is stale",
9827            ));
9828        }
9829        #[cfg(test)]
9830        self.target_preparation_fault(
9831            crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9832        )?;
9833        self.graph
9834            .validate_prepared_legacy_graph_plan(&legacy_graph)
9835            .map_err(|error| {
9836                Self::preparation_stale(
9837                    formualizer_common::PreparationStaleReason::Graph,
9838                    format!("target graph preparation plan is stale: {error}"),
9839                )
9840            })?;
9841        #[cfg(test)]
9842        self.target_preparation_fault(
9843            crate::engine::target_preparation::TargetPreparationFault::Reservation,
9844        )?;
9845        self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
9846        self.formula_parse_diagnostics
9847            .try_reserve(pending_diagnostics.len())
9848            .map_err(|_| {
9849                resource(
9850                    formualizer_common::ResourceExhaustionReason::Admission,
9851                    pending_diagnostics.len() as u64,
9852                    pending_diagnostics.len() as u64,
9853                )
9854            })?;
9855        self.target_preparation_checkpoint(options.deadline, 0)?;
9856        #[cfg(test)]
9857        self.target_preparation_fault(
9858            crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
9859        )?;
9860
9861        let commit_started = crate::instant::FzInstant::now();
9862        let committed = self
9863            .graph
9864            .apply_prevalidated_legacy_graph_plan(legacy_graph);
9865        for formula in &prepared {
9866            let removed = self
9867                .staged_formulas
9868                .get_mut(&formula.sheet)
9869                .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
9870            debug_assert!(removed.is_some());
9871            let index_removed = self.staged_formula_index.remove(
9872                &formula.sheet,
9873                formula.lease.row,
9874                formula.lease.col,
9875            );
9876            debug_assert!(index_removed);
9877        }
9878        for package in &prepared_packages {
9879            if let Some(points) = &package.selected_points {
9880                let staged = self.staged_formulas.get_mut(&package.sheet).unwrap();
9881                let source = staged.deferred_package.as_mut().unwrap();
9882                source.consumed_engine = Some(Arc::clone(&self.source_formula_token));
9883                // The revision-validated graph commit establishes this exact source
9884                // ownership proof. Later edits retain the exclusion, not old text.
9885                source
9886                    .consumed_members
9887                    .extend(package.replay_records.iter().filter_map(|record| {
9888                        record
9889                            .partition_owner
9890                            .or(record.family)
9891                            .filter(|owner| residual_owners[&package.sheet].contains(owner))
9892                            .map(|owner| {
9893                                (
9894                                    owner,
9895                                    crate::engine::SourceCoord {
9896                                        row: record.row - 1,
9897                                        col: record.col - 1,
9898                                    },
9899                                )
9900                            })
9901                    }));
9902                source.suppressed.extend(points.iter().copied());
9903                source.source_accounted = true;
9904                self.staged_formula_index
9905                    .consume_package_points(&package.sheet, points);
9906                if source.suppressed.len() >= source.source_coordinates.len() {
9907                    staged.deferred_package = None;
9908                    self.staged_formula_index.set_package(&package.sheet, None);
9909                }
9910            } else {
9911                let removed = self
9912                    .staged_formulas
9913                    .get_mut(&package.sheet)
9914                    .and_then(|staged| staged.deferred_package.take());
9915                debug_assert!(removed.is_some());
9916                self.staged_formula_index.set_package(&package.sheet, None);
9917            }
9918        }
9919        for (sheet, (families, partitions)) in residual_sources {
9920            if let Some(source) = self
9921                .staged_formulas
9922                .get_mut(&sheet)
9923                .and_then(|staged| staged.deferred_package.as_mut())
9924            {
9925                source.families = families;
9926                source.partitioned_families = partitions;
9927                source
9928                    .consumed_members
9929                    .retain(|(owner, _)| residual_owners[&sheet].contains(owner));
9930                self.staged_formula_index.update_package_family_count(
9931                    &sheet,
9932                    source.families.len() + source.partitioned_families.len(),
9933                );
9934            }
9935        }
9936        let empty_sheets = self
9937            .staged_formulas
9938            .iter()
9939            .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
9940            .collect::<Vec<_>>();
9941        for sheet in empty_sheets {
9942            self.staged_formulas.remove(&sheet);
9943        }
9944        if committed > 0 {
9945            self.mark_topology_edited();
9946        }
9947        self.formula_parse_diagnostics.extend(pending_diagnostics);
9948        if !prepared.is_empty() || !prepared_packages.is_empty() {
9949            let mut ingest_delta = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
9950            ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
9951                prepared_packages
9952                    .iter()
9953                    .map(|package| {
9954                        (package.replay_records.len() as u64).saturating_add(
9955                            if package.selected_points.is_some() {
9956                                package.direct_cells
9957                            } else {
9958                                0
9959                            },
9960                        )
9961                    })
9962                    .sum::<u64>(),
9963            );
9964            ingest_delta.graph_formula_cells_materialized = committed as u64;
9965            ingest_delta.graph_vertices_created = new_vertices as u64;
9966            ingest_delta.graph_edges_created = new_edges as u64;
9967            for package in &prepared_packages {
9968                let source = &package.source_report;
9969                ingest_delta.source_formula_events = ingest_delta
9970                    .source_formula_events
9971                    .saturating_add(source.source_formula_events);
9972                ingest_delta.source_formula_records_spooled = ingest_delta
9973                    .source_formula_records_spooled
9974                    .saturating_add(source.source_formula_records_spooled);
9975                ingest_delta.source_spool_encoded_bytes = ingest_delta
9976                    .source_spool_encoded_bytes
9977                    .saturating_add(source.source_spool_encoded_bytes);
9978                ingest_delta.source_spool_peak_memory_bytes = ingest_delta
9979                    .source_spool_peak_memory_bytes
9980                    .max(source.source_spool_peak_memory_bytes);
9981                ingest_delta.source_spool_spilled_bytes = ingest_delta
9982                    .source_spool_spilled_bytes
9983                    .saturating_add(source.source_spool_spilled_bytes);
9984                ingest_delta.source_spool_spill_files = ingest_delta
9985                    .source_spool_spill_files
9986                    .saturating_add(source.source_spool_spill_files);
9987                ingest_delta.source_spool_replays = ingest_delta
9988                    .source_spool_replays
9989                    .saturating_add(source.source_spool_replays)
9990                    .saturating_add(package.spool_replays);
9991                ingest_delta.source_families_seen = ingest_delta
9992                    .source_families_seen
9993                    .saturating_add(source.families_seen);
9994                ingest_delta.source_family_cells_seen = ingest_delta
9995                    .source_family_cells_seen
9996                    .saturating_add(source.family_cells_seen);
9997                ingest_delta.source_family_shadow_eligible = ingest_delta
9998                    .source_family_shadow_eligible
9999                    .saturating_add(source.source_clean_families);
10000                ingest_delta.source_family_shadow_eligible_cells = ingest_delta
10001                    .source_family_shadow_eligible_cells
10002                    .saturating_add(source.source_clean_cells);
10003                ingest_delta.source_partitioned_families_seen = ingest_delta
10004                    .source_partitioned_families_seen
10005                    .saturating_add(source.source_fragmentable_families);
10006                ingest_delta.source_partition_holes = ingest_delta
10007                    .source_partition_holes
10008                    .saturating_add(source.source_hole_exclusions);
10009                ingest_delta.source_partition_ordinary_exceptions = ingest_delta
10010                    .source_partition_ordinary_exceptions
10011                    .saturating_add(source.source_ordinary_exclusions);
10012                ingest_delta.source_partition_surviving_cells = ingest_delta
10013                    .source_partition_surviving_cells
10014                    .saturating_add(source.source_fragmentable_cells);
10015                for (reason, count) in &source.fallback_reasons {
10016                    let total = ingest_delta
10017                        .fallback_reasons
10018                        .entry(reason.clone())
10019                        .or_default();
10020                    *total = total.saturating_add(*count);
10021                }
10022
10023                ingest_delta.source_family_fallback = ingest_delta
10024                    .source_family_fallback
10025                    .saturating_add(source.families_seen);
10026                ingest_delta.source_family_fallback_cells = ingest_delta
10027                    .source_family_fallback_cells
10028                    .saturating_add(source.family_cells_seen);
10029            }
10030            self.record_formula_ingest_report(ingest_delta);
10031        }
10032        let commit_window = commit_started.elapsed();
10033        let retained_cells = self
10034            .staged_formula_index
10035            .all_leases()
10036            .into_iter()
10037            .filter_map(|(sheet, lease)| {
10038                formualizer_common::RangeAddress::new(
10039                    sheet, lease.row, lease.col, lease.row, lease.col,
10040                )
10041                .ok()
10042            })
10043            .collect::<Vec<_>>();
10044        let observed_scratch_bytes = self
10045            .active_resource_ledger
10046            .as_ref()
10047            .map_or(0, |ledger| ledger.snapshot().scratch_peak)
10048            .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
10049        let committed_spans = 0u64;
10050        let selected_source_families = prepared_packages
10051            .iter()
10052            .filter(|package| package.selected_points.is_none())
10053            .map(|package| package.lease.family_count)
10054            .sum::<usize>()
10055            + prepared_packages
10056                .iter()
10057                .filter(|package| package.selected_points.is_some())
10058                .flat_map(|package| package.replay_records.iter())
10059                .filter_map(|record| record.partition_owner.or(record.family))
10060                .chain(
10061                    prepared_packages
10062                        .iter()
10063                        .flat_map(|package| package.complete_selections.iter().copied()),
10064                )
10065                .collect::<BTreeSet<_>>()
10066                .len();
10067        let selected_staged_cells = prepared.len().saturating_add(
10068            prepared_packages
10069                .iter()
10070                .map(|package| {
10071                    package
10072                        .selected_points
10073                        .as_ref()
10074                        .map_or(package.replay_records.len(), BTreeSet::len)
10075                })
10076                .sum::<usize>(),
10077        );
10078        let actual_commit_work = (new_vertices as u64)
10079            .saturating_add(new_edges as u64)
10080            .saturating_add(committed as u64)
10081            .saturating_add(committed_spans)
10082            .saturating_add(prepared.len() as u64)
10083            .saturating_add(
10084                prepared_packages
10085                    .iter()
10086                    .map(|package| {
10087                        package
10088                            .selected_points
10089                            .as_ref()
10090                            .map_or(1, |points| points.len() as u64)
10091                    })
10092                    .sum::<u64>(),
10093            );
10094        let report = PreparedTargetGraphReport {
10095            request_id: request_id.unwrap_or_default(),
10096            requested_targets: targets.len(),
10097            normalized_regions: visited_regions.len(),
10098            normalized_target_list: normalized,
10099            selected_staged_cells,
10100            selected_source_families,
10101            retained_staged_cells: self.staged_formula_count(),
10102            selected_cells,
10103            retained_cells,
10104            widened_scope: scope,
10105            widening_reasons: reasons,
10106            revisions: assumptions,
10107            commit_window,
10108            estimated_scratch_bytes: scratch_bytes,
10109            observed_scratch_bytes,
10110            estimated_commit_work: (new_vertices as u64)
10111                .saturating_add(new_edges as u64)
10112                .saturating_add(prepared.len() as u64)
10113                .saturating_add(selected_package_records),
10114            actual_commit_work,
10115            outcome: PreparationOutcome::Prepared,
10116        };
10117        self.observe_target_preparation_report(&report);
10118        Ok(report)
10119    }
10120
10121    /// Build graph for all staged formulas.
10122    pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
10123        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
10124            engine.build_graph_all_unobserved()
10125        })
10126    }
10127
10128    fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
10129        let selected = self.staged_formula_count();
10130        let started = crate::instant::FzInstant::now();
10131        self.resource_checkpoint(selected as u64)?;
10132        let scratch_bytes = (selected as u64).saturating_mul(256);
10133        let result = self.with_request_scratch(scratch_bytes, |engine| {
10134            let index_snapshot = engine.staged_formula_index.clone();
10135            let collected = std::mem::take(&mut engine.staged_formulas)
10136                .into_iter()
10137                .collect();
10138            engine.staged_formula_index.clear_all();
10139            engine.build_graph_from_staged_batches(collected, false, index_snapshot)
10140        });
10141        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
10142        result
10143    }
10144
10145    /// Build graph for specific sheets (consuming only those staged entries).
10146    pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
10147        &mut self,
10148        sheets: I,
10149    ) -> Result<(), formualizer_parse::ExcelError> {
10150        let mut sheets = sheets.into_iter();
10151        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
10152            let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
10153            engine.with_request_scratch(name_scratch, |engine| {
10154                // Allocation failure follows the baseline process-fatal policy; it is not a
10155                // recoverable resource error or a new staged-preparation route.
10156                let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
10157                engine.charge_bounded_work(names.len() as u64)?;
10158                engine.build_graph_for_sheet_names_unobserved(names)
10159            })
10160        })
10161    }
10162
10163    fn build_graph_for_sheet_names_unobserved(
10164        &mut self,
10165        sheets: Vec<String>,
10166    ) -> Result<(), formualizer_parse::ExcelError> {
10167        let started = crate::instant::FzInstant::now();
10168        let selected = sheets
10169            .iter()
10170            .filter_map(|sheet| self.staged_formulas.get(sheet))
10171            .map(StagedSheet::len)
10172            .sum::<usize>();
10173        self.resource_checkpoint(selected as u64)?;
10174        let scratch_bytes = (selected as u64).saturating_mul(256);
10175        self.reserve_request_scratch(scratch_bytes)?;
10176        let index_snapshot = self.staged_formula_index.clone();
10177        let mut collected = Vec::new();
10178        for sheet in sheets {
10179            if let Some(staged) = self.staged_formulas.remove(&sheet) {
10180                self.index_removed_staged_sheet(&sheet, &staged);
10181                collected.push((sheet, staged));
10182            }
10183        }
10184        let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
10185        self.release_request_scratch(scratch_bytes);
10186        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
10187        result
10188    }
10189
10190    fn build_graph_from_staged_batches(
10191        &mut self,
10192        collected: StagedFormulaBatches,
10193        share_parse_cache_across_sheets: bool,
10194        staged_index_snapshot: StagedFormulaIndex,
10195    ) -> Result<(), formualizer_parse::ExcelError> {
10196        if collected.is_empty() {
10197            return Ok(());
10198        }
10199        for (sheet, _) in &collected {
10200            let _ = self.add_sheet(sheet);
10201        }
10202
10203        let diagnostics_len = self.formula_parse_diagnostics.len();
10204        let mut collected = collected;
10205        let prepared = match self
10206            .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
10207        {
10208            Ok(prepared) => prepared,
10209            Err(error) => {
10210                self.formula_parse_diagnostics.truncate(diagnostics_len);
10211                for (sheet, staged) in collected {
10212                    self.restore_staged_sheet(sheet, staged);
10213                }
10214                self.staged_formula_index = staged_index_snapshot;
10215                return Err(error);
10216            }
10217        };
10218        let (ordinary, compressed, direct, may_fail) = prepared;
10219
10220        // A first build (no formula in the graph yet) goes through the eager
10221        // first load's machinery: the builder pre-allocates each column's
10222        // targets as one id run, installs family members as virtual runs and
10223        // builds the authority once at the end (decision 26).
10224        // The first load's builder plans and applies one chunk at a time, so
10225        // a planning error would leave earlier chunks in the graph; the
10226        // incremental path plans everything first. Check every distinct
10227        // formula first, in the incremental path's order (it reports the
10228        // same first error and leaves the graph untouched).
10229        let first_build = if self.deferred_build_can_be_first_load() {
10230            if let Err(error) = self.check_staged_formula_plans(
10231                ordinary
10232                    .iter()
10233                    .chain(compressed.iter().map(|(batch, _)| batch)),
10234                &may_fail,
10235            ) {
10236                self.formula_parse_diagnostics.truncate(diagnostics_len);
10237                for (sheet, staged) in collected {
10238                    self.restore_staged_sheet(sheet, staged);
10239                }
10240                self.staged_formula_index = staged_index_snapshot;
10241                return Err(error);
10242            }
10243            self.deferred_build_as_first_load()
10244        } else {
10245            None
10246        };
10247        let built_sheets: Vec<String> = if first_build.is_some() {
10248            collected.iter().map(|(sheet, _)| sheet.clone()).collect()
10249        } else {
10250            Vec::new()
10251        };
10252
10253        // Keep the original source/spool alive through every fallible ingestion route.
10254        // Graph admission may have committed a prefix; replay replaces those placements
10255        // rather than treating their cached values as authoritative source.
10256        let result = (|| {
10257            if !ordinary.is_empty() {
10258                self.ingest_formula_batches(ordinary)?;
10259            }
10260            if !compressed.is_empty() {
10261                self.ingest_compressed_formula_source_batches(compressed)?;
10262            }
10263            if !direct.is_empty() {
10264                self.finish_compressed_formula_sources(direct)?;
10265            }
10266            Ok(())
10267        })();
10268        if let Some(saved) = first_build {
10269            self.leave_deferred_first_load(saved, &built_sheets);
10270        }
10271        if let Err(error) = result {
10272            self.formula_parse_diagnostics.truncate(diagnostics_len);
10273            for (sheet, staged) in collected {
10274                self.restore_staged_sheet(sheet, staged);
10275            }
10276            self.staged_formula_index = staged_index_snapshot;
10277            return Err(error);
10278        }
10279        self.dedup_formula_parse_diagnostics_since(diagnostics_len);
10280        Ok(())
10281    }
10282
10283    /// Enter the first-load ingest mode for a deferred build when the graph
10284    /// holds no formula yet and no load is in progress: the settings the
10285    /// calamine loader uses for its eager first load (lazy sheet index, no
10286    /// small-range expansion, first-load fast path). Returns the settings
10287    /// to restore, or `None` when the build takes the incremental path.
10288    fn deferred_build_can_be_first_load(&self) -> bool {
10289        !self.graph.first_load_assume_new()
10290            && !self.graph_admission_enabled()
10291            && self.graph.formula_vertex_count() == 0
10292    }
10293
10294    /// Plan each distinct formula of `batches` that may fail planning once
10295    /// (a member is planned through its template, staged before it) and
10296    /// drop the plans: the first planning error, in the order the
10297    /// incremental ingest meets it.
10298    fn check_staged_formula_plans<'b>(
10299        &mut self,
10300        batches: impl Iterator<Item = &'b FormulaIngestBatch>,
10301        may_fail: &FxHashSet<crate::engine::arena::AstNodeId>,
10302    ) -> Result<(), ExcelError> {
10303        if may_fail.is_empty() {
10304            return Ok(());
10305        }
10306        let mut seen: FxHashSet<(SheetId, crate::engine::arena::AstNodeId)> = FxHashSet::default();
10307        for batch in batches {
10308            let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
10309                ExcelError::new(ExcelErrorKind::Ref)
10310                    .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
10311            })?;
10312            let mut pipeline = self.ingest_pipeline();
10313            for record in &batch.formulas {
10314                if record.member_anchor.is_some()
10315                    || !may_fail.contains(&record.ast_id)
10316                    || !seen.insert((sheet_id, record.ast_id))
10317                {
10318                    continue;
10319                }
10320                let placement = CellRef::new(
10321                    sheet_id,
10322                    Coord::from_excel(record.row, record.col, true, true),
10323                );
10324                pipeline.ingest_formula(
10325                    FormulaAstInput::RawArena(record.ast_id),
10326                    placement,
10327                    None,
10328                )?;
10329            }
10330        }
10331        Ok(())
10332    }
10333
10334    fn deferred_build_as_first_load(&mut self) -> Option<(crate::engine::SheetIndexMode, usize)> {
10335        if !self.deferred_build_can_be_first_load() {
10336            return None;
10337        }
10338        let saved = (
10339            self.graph.get_config().sheet_index_mode,
10340            self.config.range_expansion_limit,
10341        );
10342        self.graph
10343            .set_sheet_index_mode(crate::engine::SheetIndexMode::Lazy);
10344        self.config.range_expansion_limit = 0;
10345        self.graph.set_first_load_assume_new(true);
10346        self.graph.reset_ensure_touched();
10347        Some(saved)
10348    }
10349
10350    /// Leave the first-load mode of [`Self::deferred_build_as_first_load`]:
10351    /// the authority is built once here, as at the end of an eager load.
10352    fn leave_deferred_first_load(
10353        &mut self,
10354        (index_mode, range_limit): (crate::engine::SheetIndexMode, usize),
10355        sheets: &[String],
10356    ) {
10357        self.graph.set_first_load_assume_new(false);
10358        self.graph.reset_ensure_touched();
10359        self.graph.set_sheet_index_mode(index_mode);
10360        self.config.range_expansion_limit = range_limit;
10361        for sheet in sheets {
10362            self.graph.finalize_sheet_index(sheet);
10363        }
10364    }
10365
10366    fn prepare_staged_formula_batches(
10367        &mut self,
10368        collected: &mut StagedFormulaBatches,
10369        share_parse_cache_across_sheets: bool,
10370    ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
10371        let mut ordinary = Vec::new();
10372        let mut compressed = Vec::new();
10373        let mut direct = Vec::new();
10374        let mut may_fail = FxHashSet::default();
10375        let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
10376            rustc_hash::FxHashMap::default();
10377        cache.reserve(4096);
10378
10379        for (sheet, staged) in collected {
10380            if !share_parse_cache_across_sheets {
10381                cache.clear();
10382            }
10383            // Load-time family grouping, as the eager first load does:
10384            // relative copies of the formula above (or to the left) become
10385            // members of its family and are never interned.
10386            let mut grouper = crate::engine::FormulaFamilyGrouper::new();
10387            // The staged texts, then the deferred package's replayed ones,
10388            // are walked once (no combined copy: a replay can be the whole
10389            // sheet).
10390            let mut replayed_records = Vec::new();
10391            let deferred_source = None;
10392            let mut deferred_fallback = None;
10393            if let Some(package) = staged.deferred_package.as_mut() {
10394                if package.sheet_name != *sheet {
10395                    return Err(ExcelError::new(ExcelErrorKind::Value)
10396                        .with_message("deferred formula package sheet mismatch"));
10397                }
10398                let eligible: Vec<_> = package
10399                    .families
10400                    .iter()
10401                    .filter(|family| !package.invalidated.contains(&family.source_id))
10402                    .cloned()
10403                    .collect();
10404                let eligible_partitions: Vec<_> = package
10405                    .partitioned_families
10406                    .iter()
10407                    .filter(|family| !package.invalidated.contains(&family.source_id))
10408                    .cloned()
10409                    .collect();
10410
10411                let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
10412                for partition in &eligible_partitions {
10413                    replay_disposition
10414                        .register_partition(partition, false)
10415                        .map_err(|reason| {
10416                            ExcelError::new(ExcelErrorKind::Value).with_message(reason)
10417                        })?;
10418                }
10419                replay_disposition
10420                    .extend_suppressed_excel_coords(package.suppressed.iter().copied());
10421                let replayed = package
10422                    .replay
10423                    .lock()
10424                    .map_err(|_| {
10425                        ExcelError::new(ExcelErrorKind::Value)
10426                            .with_message("deferred formula spool lock poisoned")
10427                    })?
10428                    .replay(&replay_disposition)
10429                    .map_err(|message| {
10430                        ExcelError::new(ExcelErrorKind::Value).with_message(message)
10431                    })?;
10432                replayed_records = replayed;
10433                let mut report = package.accounting_report();
10434                report.source_spool_replays = report.source_spool_replays.saturating_add(1);
10435                deferred_fallback = Some((report, package.families.clone(), eligible_partitions));
10436            }
10437
10438            let n_entries = staged.entries.len() + replayed_records.len();
10439            let entries = staged
10440                .entries
10441                .iter()
10442                .cloned()
10443                .map(|(row, col, text)| (row, col, text, None))
10444                .chain(replayed_records.into_iter().map(|record| {
10445                    (
10446                        record.row,
10447                        record.col,
10448                        record.text,
10449                        Some((record.source_order, record.family, record.partition_owner)),
10450                    )
10451                }));
10452            let mut formulas = Vec::with_capacity(n_entries);
10453            let staged_order_base = u64::MAX.saturating_sub(n_entries as u64);
10454            for (entry_index, (row, col, txt, source_proof)) in entries.enumerate() {
10455                let key = if txt.starts_with('=') {
10456                    txt
10457                } else {
10458                    format!("={txt}")
10459                };
10460                grouper.note_formula();
10461                // A shared-formula family's records name their family.
10462                let shared_key = source_proof
10463                    .and_then(|(_, family, _)| family)
10464                    .map(|family| u64::from(family.source_index));
10465                let staged_record = if let Some(cached) = cache.get(&key) {
10466                    grouper.note_parse_cache_hit();
10467                    cached.map(|ast_id| {
10468                        self.note_staged_formula(&mut grouper, row, col, ast_id);
10469                        FormulaIngestRecord::new(row, col, ast_id, Some(Arc::<str>::from(key)))
10470                    })
10471                } else if let Some(record) =
10472                    self.stage_relocated_text(&mut grouper, row, col, &key, shared_key)
10473                {
10474                    Some(record)
10475                } else {
10476                    grouper.note_parse(key.len());
10477                    let parsed = formualizer_parse::parser::parse(&key);
10478                    let parsed_text = parsed.is_ok();
10479                    let parsed = match parsed {
10480                        Ok(parsed) => Some(parsed),
10481                        Err(error) => self.handle_formula_parse_error(
10482                            sheet,
10483                            row,
10484                            col,
10485                            &key,
10486                            error.to_string(),
10487                        )?,
10488                    };
10489                    match parsed {
10490                        Some(ast) => {
10491                            let record = self.stage_formula_ast(&mut grouper, row, col, &ast, None);
10492                            if parsed_text {
10493                                self.note_parsed_text(
10494                                    &mut grouper,
10495                                    row,
10496                                    col,
10497                                    &key,
10498                                    &ast,
10499                                    &record,
10500                                    shared_key,
10501                                );
10502                            }
10503                            if !record.is_family_member() && self.formula_may_fail_planning(&ast) {
10504                                may_fail.insert(record.ast_id);
10505                            }
10506                            // A member's text is not worth caching: relative
10507                            // copies do not repeat their text.
10508                            if record.is_family_member() {
10509                                Some(record)
10510                            } else {
10511                                let ast_id = record.ast_id;
10512                                cache.insert(key.clone(), Some(ast_id));
10513                                Some(FormulaIngestRecord::new(
10514                                    row,
10515                                    col,
10516                                    ast_id,
10517                                    Some(Arc::<str>::from(key)),
10518                                ))
10519                            }
10520                        }
10521                        None => {
10522                            cache.insert(key, None);
10523                            None
10524                        }
10525                    }
10526                };
10527
10528                if let Some(mut formula) = staged_record {
10529                    if let Some((order, family, owner)) = source_proof {
10530                        formula = formula.with_source_proof(order, family, owner);
10531                    } else if deferred_source.is_some() {
10532                        formula = formula.with_source_proof(
10533                            crate::engine::SourceFormulaOrder::new(
10534                                staged_order_base.saturating_add(entry_index as u64),
10535                            ),
10536                            None,
10537                            None,
10538                        );
10539                    }
10540                    formulas.push(formula);
10541                }
10542            }
10543
10544            self.finish_family_grouper(&mut grouper);
10545            let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
10546            if let Some((report, preparation)) = deferred_source {
10547                direct.push((batch, report, preparation));
10548            } else if let Some((report, families, partitions)) = deferred_fallback {
10549                let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
10550                    batch.sheet_name.clone(),
10551                    report,
10552                    families,
10553                    partitions,
10554                );
10555                compressed.push((batch, source_batch));
10556            } else if !batch.is_empty() {
10557                ordinary.push(batch);
10558            }
10559        }
10560        if std::env::var("FZ_DEBUG_LOAD")
10561            .ok()
10562            .is_some_and(|v| v != "0")
10563        {
10564            eprintln!(
10565                "[fz][families] deferred build: {}",
10566                self.source_family_counters.debug_line()
10567            );
10568        }
10569        Ok((ordinary, compressed, direct, may_fail))
10570    }
10571
10572    /// Whether planning `ast` can fail: a reference to a sheet that does not
10573    /// exist, an external, 3-D or table reference, or a reversed range.
10574    /// Unqualified cell and range references, names, literals and calls
10575    /// always plan.
10576    fn formula_may_fail_planning(&self, ast: &formualizer_parse::parser::ASTNode) -> bool {
10577        use formualizer_parse::parser::{ASTNodeType, ReferenceType};
10578        let sheet_missing = |sheet: &Option<String>| {
10579            sheet
10580                .as_deref()
10581                .is_some_and(|s| self.graph.sheet_id(s).is_none())
10582        };
10583        match &ast.node_type {
10584            ASTNodeType::Literal(_) | ASTNodeType::Omitted => false,
10585            ASTNodeType::Reference { reference, .. } => match reference {
10586                ReferenceType::Cell { sheet, .. } => sheet_missing(sheet),
10587                ReferenceType::Range {
10588                    sheet,
10589                    start_row,
10590                    start_col,
10591                    end_row,
10592                    end_col,
10593                    ..
10594                } => {
10595                    sheet_missing(sheet)
10596                        || matches!((start_row, end_row), (Some(a), Some(b)) if a > b)
10597                        || matches!((start_col, end_col), (Some(a), Some(b)) if a > b)
10598                }
10599                ReferenceType::NamedRange(_) => false,
10600                _ => true,
10601            },
10602            ASTNodeType::UnaryOp { expr, .. } => self.formula_may_fail_planning(expr),
10603            ASTNodeType::BinaryOp { left, right, .. } => {
10604                self.formula_may_fail_planning(left) || self.formula_may_fail_planning(right)
10605            }
10606            ASTNodeType::Function { args, .. } => {
10607                args.iter().any(|a| self.formula_may_fail_planning(a))
10608            }
10609            ASTNodeType::Call { callee, args } => {
10610                self.formula_may_fail_planning(callee)
10611                    || args.iter().any(|a| self.formula_may_fail_planning(a))
10612            }
10613            ASTNodeType::Array(rows) => rows
10614                .iter()
10615                .flatten()
10616                .any(|a| self.formula_may_fail_planning(a)),
10617        }
10618    }
10619
10620    /// Begin bulk Arrow ingest for base values (Phase A)
10621    pub fn begin_bulk_ingest_arrow(
10622        &mut self,
10623    ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
10624        crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
10625    }
10626
10627    /// Begin bulk updates to Arrow store (Phase C)
10628    pub fn begin_bulk_update_arrow(
10629        &mut self,
10630    ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
10631        crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
10632    }
10633
10634    fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
10635        self.graph.sheet_id(sheet).ok_or(
10636            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
10637                name: sheet.to_string(),
10638                reason: "Unknown sheet".to_string(),
10639            },
10640        )
10641    }
10642
10643    fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
10644        if row_1based == 0 {
10645            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10646        }
10647        Ok(row_1based - 1)
10648    }
10649
10650    fn normalize_row_range_1based(
10651        start_row_1based: u32,
10652        end_row_1based: u32,
10653    ) -> Result<(u32, u32), crate::engine::EditorError> {
10654        if start_row_1based == 0 || end_row_1based == 0 {
10655            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10656        }
10657        if start_row_1based > end_row_1based {
10658            return Err(crate::engine::EditorError::TransactionFailed {
10659                reason: "Row range start is greater than end".to_string(),
10660            });
10661        }
10662        Ok((start_row_1based - 1, end_row_1based - 1))
10663    }
10664
10665    fn invalidate_row_visibility_mask_cache(&self) {
10666        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
10667            cache.clear();
10668        }
10669    }
10670
10671    /// Hidden rows changed: drop the cached masks and dirty the formulas
10672    /// that read row visibility (SUBTOTAL/AGGREGATE).
10673    fn row_visibility_changed(&mut self) {
10674        self.invalidate_row_visibility_mask_cache();
10675        self.graph.mark_row_visibility_readers_dirty();
10676    }
10677
10678    fn set_row_hidden_by_sheet_id(
10679        &mut self,
10680        sheet_id: SheetId,
10681        row0: u32,
10682        hidden: bool,
10683        source: RowVisibilitySource,
10684    ) -> bool {
10685        let changed = {
10686            let state = self.row_visibility.entry(sheet_id).or_default();
10687            state.set_row_hidden(row0, hidden, source)
10688        };
10689
10690        let remove_entry = self
10691            .row_visibility
10692            .get(&sheet_id)
10693            .map(|state| state.is_empty())
10694            .unwrap_or(false);
10695        if remove_entry {
10696            self.row_visibility.remove(&sheet_id);
10697        }
10698
10699        if changed {
10700            self.row_visibility_changed();
10701        }
10702
10703        changed
10704    }
10705
10706    fn set_rows_hidden_by_sheet_id(
10707        &mut self,
10708        sheet_id: SheetId,
10709        start_row0: u32,
10710        end_row0: u32,
10711        hidden: bool,
10712        source: RowVisibilitySource,
10713    ) -> bool {
10714        let changed = {
10715            let state = self.row_visibility.entry(sheet_id).or_default();
10716            state.set_rows_hidden(start_row0, end_row0, hidden, source)
10717        };
10718
10719        let remove_entry = self
10720            .row_visibility
10721            .get(&sheet_id)
10722            .map(|state| state.is_empty())
10723            .unwrap_or(false);
10724        if remove_entry {
10725            self.row_visibility.remove(&sheet_id);
10726        }
10727
10728        if changed {
10729            self.row_visibility_changed();
10730        }
10731
10732        changed
10733    }
10734
10735    fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
10736        if count == 0 {
10737            return;
10738        }
10739        let mut changed = false;
10740        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10741            changed = state.insert_rows(before0, count);
10742            state.is_empty()
10743        } else {
10744            false
10745        };
10746        if remove_entry {
10747            self.row_visibility.remove(&sheet_id);
10748        }
10749        if changed {
10750            self.row_visibility_changed();
10751        }
10752    }
10753
10754    fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
10755        if count == 0 {
10756            return;
10757        }
10758        let mut changed = false;
10759        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10760            changed = state.delete_rows(start0, count);
10761            state.is_empty()
10762        } else {
10763            false
10764        };
10765        if remove_entry {
10766            self.row_visibility.remove(&sheet_id);
10767        }
10768        if changed {
10769            self.row_visibility_changed();
10770        }
10771    }
10772
10773    fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10774        if let crate::engine::ChangeEvent::SetRowVisibility {
10775            sheet_id,
10776            row0,
10777            source,
10778            old_hidden,
10779            ..
10780        } = event
10781        {
10782            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
10783        }
10784    }
10785
10786    fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10787        if let crate::engine::ChangeEvent::SetRowVisibility {
10788            sheet_id,
10789            row0,
10790            source,
10791            new_hidden,
10792            ..
10793        } = event
10794        {
10795            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
10796        }
10797    }
10798
10799    fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10800        for event in events.iter().rev() {
10801            self.apply_inverse_row_visibility_event(event);
10802        }
10803    }
10804
10805    fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10806        for event in events {
10807            self.apply_forward_row_visibility_event(event);
10808        }
10809    }
10810
10811    fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10812        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10813            sheet,
10814            row,
10815            col,
10816            old,
10817            ..
10818        } = event
10819        {
10820            self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
10821        }
10822    }
10823
10824    fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10825        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10826            sheet,
10827            row,
10828            col,
10829            new,
10830            ..
10831        } = event
10832        {
10833            self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
10834        }
10835    }
10836
10837    /// Set a single cell's staged formula text to `target` (clearing it when
10838    /// `None`). Used by undo/redo replay of per-cell staged-formula deltas.
10839    fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
10840        match target {
10841            Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
10842            None => {
10843                self.clear_staged_formula_text(sheet, row, col);
10844            }
10845        }
10846    }
10847
10848    pub fn set_row_hidden(
10849        &mut self,
10850        sheet: &str,
10851        row_1based: u32,
10852        hidden: bool,
10853        source: RowVisibilitySource,
10854    ) -> Result<(), crate::engine::EditorError> {
10855        self.observe_function_semantic_epoch()
10856            .map_err(crate::engine::EditorError::Excel)?;
10857        self.observe_function_semantic_epoch()
10858            .map_err(crate::engine::EditorError::Excel)?;
10859        let sheet_id = self.ensure_known_sheet_id(sheet)?;
10860        let row0 = Self::normalize_row_1based(row_1based)?;
10861        if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
10862            self.record_structural_change(StructuralScope::Region(Region::whole_row(
10863                sheet_id, row0,
10864            )));
10865            self.mark_data_edited();
10866        }
10867        Ok(())
10868    }
10869
10870    pub fn set_rows_hidden(
10871        &mut self,
10872        sheet: &str,
10873        start_row_1based: u32,
10874        end_row_1based: u32,
10875        hidden: bool,
10876        source: RowVisibilitySource,
10877    ) -> Result<(), crate::engine::EditorError> {
10878        let sheet_id = self.ensure_known_sheet_id(sheet)?;
10879        let (start_row0, end_row0) =
10880            Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
10881        if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
10882            if start_row0 == end_row0 {
10883                self.record_structural_change(StructuralScope::Region(Region::whole_row(
10884                    sheet_id, start_row0,
10885                )));
10886            } else {
10887                self.record_structural_change(StructuralScope::Sheet(sheet_id));
10888            }
10889            self.mark_data_edited();
10890        }
10891        Ok(())
10892    }
10893
10894    pub fn is_row_hidden(
10895        &self,
10896        sheet: &str,
10897        row_1based: u32,
10898        source: Option<RowVisibilitySource>,
10899    ) -> Option<bool> {
10900        let sheet_id = self.graph.sheet_id(sheet)?;
10901        let row0 = row_1based.checked_sub(1)?;
10902        Some(
10903            self.row_visibility
10904                .get(&sheet_id)
10905                .map(|state| state.is_row_hidden(row0, source))
10906                .unwrap_or(false),
10907        )
10908    }
10909
10910    pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
10911        let sheet_id = self.graph.sheet_id(sheet)?;
10912        Some(
10913            self.row_visibility
10914                .get(&sheet_id)
10915                .map(|state| state.version())
10916                .unwrap_or(0),
10917        )
10918    }
10919
10920    /// Cells of a sheet-backed `view` whose formula (own or a family
10921    /// template) calls SUBTOTAL, or AGGREGATE when `include_aggregate`, as
10922    /// sorted, disjoint `(col, first_row, last_row)` offset intervals within
10923    /// the view. Each formula's call bits are precomputed in the AST arena;
10924    /// a filled-down family is checked once per run, not once per cell.
10925    fn nested_subtotal_cells_for_view(
10926        &self,
10927        view: &RangeView<'_>,
10928        include_aggregate: bool,
10929    ) -> Option<Vec<(usize, usize, usize)>> {
10930        if !view.is_sheet_backed() {
10931            return None;
10932        }
10933        let (rows, cols) = view.dims();
10934        if rows == 0 || cols == 0 {
10935            return Some(Vec::new());
10936        }
10937        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10938        let (r0, c0) = (view.start_row(), view.start_col());
10939        let mask = if include_aggregate {
10940            crate::engine::arena::SUBTOTAL_CALL | crate::engine::arena::AGGREGATE_CALL
10941        } else {
10942            crate::engine::arena::SUBTOTAL_CALL
10943        };
10944        let ds = self.graph.data_store();
10945        let mut found: Vec<(usize, usize, usize)> = self
10946            .graph
10947            .formula_intervals_in_region(
10948                sheet_id,
10949                r0 as u32,
10950                (r0 + rows - 1) as u32,
10951                c0 as u32,
10952                (c0 + cols - 1) as u32,
10953                |template| ds.ast_subtotal_calls(template) & mask != 0,
10954            )
10955            .into_iter()
10956            .map(|(col, lo, hi)| (col as usize - c0, lo as usize - r0, hi as usize - r0))
10957            .collect();
10958        found.sort_unstable();
10959        // Merge touching or overlapping intervals of a column so a binary
10960        // search finds the one covering a cell.
10961        let mut out: Vec<(usize, usize, usize)> = Vec::with_capacity(found.len());
10962        for (col, lo, hi) in found {
10963            match out.last_mut() {
10964                Some(last) if last.0 == col && lo <= last.2 + 1 => last.2 = last.2.max(hi),
10965                _ => out.push((col, lo, hi)),
10966            }
10967        }
10968        Some(out)
10969    }
10970
10971    fn build_row_visibility_mask_for_view(
10972        &self,
10973        view: &RangeView<'_>,
10974        mode: VisibilityMaskMode,
10975    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
10976        let sheet_rows = view.sheet().nrows as usize;
10977        if sheet_rows == 0 || view.start_row() >= sheet_rows {
10978            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
10979        }
10980
10981        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10982        let start_row0 = view.start_row() as u32;
10983        let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
10984        let version = self
10985            .row_visibility
10986            .get(&sheet_id)
10987            .map(|state| state.version())
10988            .unwrap_or(0);
10989        let key = VisibilityMaskCacheKey {
10990            sheet_id,
10991            start_row0,
10992            end_row0,
10993            mode,
10994            version,
10995        };
10996
10997        if let Ok(cache) = self.row_visibility_mask_cache.read()
10998            && let Some(mask) = cache.get(&key)
10999        {
11000            #[cfg(test)]
11001            visibility_mask_test_hooks::inc_hit();
11002            return Some(mask.clone());
11003        }
11004
11005        #[cfg(test)]
11006        visibility_mask_test_hooks::inc_miss();
11007
11008        let state = self.row_visibility.get(&sheet_id);
11009        let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
11010        for row0 in start_row0..=end_row0 {
11011            let manual_hidden = state
11012                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
11013                .unwrap_or(false);
11014            let filter_hidden = state
11015                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
11016                .unwrap_or(false);
11017
11018            let include = match mode {
11019                VisibilityMaskMode::IncludeAll => true,
11020                VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
11021                VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
11022                VisibilityMaskMode::ExcludeManualOrFilterHidden => {
11023                    !(manual_hidden || filter_hidden)
11024                }
11025            };
11026            out.push(include);
11027        }
11028
11029        let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
11030        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
11031            const MAX_CACHE_ENTRIES: usize = 4096;
11032            if cache.len() >= MAX_CACHE_ENTRIES {
11033                cache.clear();
11034                #[cfg(test)]
11035                visibility_mask_test_hooks::inc_eviction();
11036            }
11037            cache.insert(key, mask.clone());
11038        }
11039
11040        Some(mask)
11041    }
11042
11043    fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
11044        let changes =
11045            crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
11046        let global_changed = changes.epoch != self.function_semantic_epoch_seen;
11047        let provider_revision = self.resolver.planning_semantic_revision();
11048        let provider_changed = provider_revision != self.function_provider_revision_seen;
11049        if !global_changed && !provider_changed {
11050            return Ok(false);
11051        }
11052
11053        let changed = !changes.keys.is_empty();
11054        if global_changed && changed || provider_changed {
11055            self.cached_static_schedule = None;
11056            self.recent_schedules.clear();
11057            self.base_schedule = None;
11058        }
11059        self.function_semantic_epoch_seen = changes.epoch;
11060        self.function_provider_revision_seen = provider_revision;
11061        Ok(false)
11062    }
11063
11064    pub(crate) fn ast_uses_changed_function(
11065        ast: &ASTNode,
11066        changed: &BTreeSet<(String, String)>,
11067    ) -> bool {
11068        match &ast.node_type {
11069            ASTNodeType::Function { name, args } => {
11070                let normalized = name.to_uppercase();
11071                let mut spellings = vec![(String::new(), normalized.clone())];
11072                let mut stripped = normalized.as_str();
11073                while let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
11074                    .iter()
11075                    .find_map(|prefix| stripped.strip_prefix(prefix))
11076                {
11077                    stripped = rest;
11078                    spellings.push((String::new(), stripped.to_string()));
11079                }
11080                let resolved = crate::function_registry::resolve("", name);
11081                let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
11082                    || resolved.as_ref().is_some_and(|resolved| {
11083                        changed.contains(&(
11084                            resolved.namespace.clone(),
11085                            resolved.canonical_name.clone(),
11086                        ))
11087                    });
11088                directly_changed
11089                    || resolved.is_none()
11090                    || args
11091                        .iter()
11092                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
11093            }
11094            ASTNodeType::Call { callee, args } => {
11095                Self::ast_uses_changed_function(callee, changed)
11096                    || args
11097                        .iter()
11098                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
11099            }
11100            ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
11101            ASTNodeType::BinaryOp { left, right, .. } => {
11102                Self::ast_uses_changed_function(left, changed)
11103                    || Self::ast_uses_changed_function(right, changed)
11104            }
11105            ASTNodeType::Array(rows) => rows
11106                .iter()
11107                .flatten()
11108                .any(|node| Self::ast_uses_changed_function(node, changed)),
11109            _ => false,
11110        }
11111    }
11112
11113    fn ast_contains_function(ast: &ASTNode) -> bool {
11114        match &ast.node_type {
11115            ASTNodeType::Function { .. } => true,
11116            ASTNodeType::Call { callee, args } => {
11117                Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
11118            }
11119            ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
11120            ASTNodeType::BinaryOp { left, right, .. } => {
11121                Self::ast_contains_function(left) || Self::ast_contains_function(right)
11122            }
11123            ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
11124            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
11125        }
11126    }
11127
11128    fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
11129        Region::rows_from(sheet_id, start_row0)
11130    }
11131
11132    fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
11133        Region::cols_from(sheet_id, start_col0)
11134    }
11135
11136    #[cfg(test)]
11137    pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
11138        self.force_non_cycle_schedule_fallback_for_test = true;
11139    }
11140
11141    fn materialize_deferred_sheet_before_structural_edit(
11142        &mut self,
11143        sheet: &str,
11144    ) -> Result<(), crate::engine::EditorError> {
11145        if self.staged_formulas.contains_key(sheet) {
11146            self.build_graph_for_sheets([sheet])?;
11147        }
11148        Ok(())
11149    }
11150
11151    /// Refuse an insert of `count` rows (`rows`) or columns at 0-based
11152    /// `before0` that would push content off the grid, as Excel does; nothing
11153    /// is changed. Content is the Arrow extent (which may include trailing
11154    /// empty cells, so this can refuse conservatively) and every formula.
11155    fn check_insert_room(
11156        &self,
11157        sheet: &str,
11158        before0: u32,
11159        count: u32,
11160        rows: bool,
11161    ) -> Result<(), crate::engine::EditorError> {
11162        let limit = if rows {
11163            crate::engine::authority::geom::MAX_ROW
11164        } else {
11165            crate::engine::authority::geom::MAX_COL
11166        } + 1;
11167        let arrow = self.arrow_sheets.sheet(sheet).map_or(0, |a| {
11168            if rows {
11169                a.nrows
11170            } else {
11171                u32::try_from(a.columns.len()).unwrap_or(u32::MAX)
11172            }
11173        });
11174        let formulas = self.graph.sheet_id(sheet).map_or(0, |id| {
11175            self.graph
11176                .grid_vertices_in_sheet(id)
11177                .map(|(_, at)| if rows { at.row() } else { at.col() } + 1)
11178                .max()
11179                .unwrap_or(0)
11180        });
11181        let extent = arrow.max(formulas);
11182        let pushed_off = extent > before0 && extent.saturating_add(count) > limit;
11183        if count > limit || pushed_off {
11184            return Err(crate::engine::EditorError::Excel(
11185                ExcelError::new(ExcelErrorKind::Ref).with_message(format!(
11186                    "inserting {count} {} would push non-empty cells off the sheet",
11187                    if rows { "rows" } else { "columns" }
11188                )),
11189            ));
11190        }
11191        Ok(())
11192    }
11193
11194    fn structural_row_occupancy(
11195        &self,
11196        sheet: &str,
11197        sheet_id: SheetId,
11198    ) -> crate::engine::graph::StructuralOccupancy {
11199        if !self.graph.has_compressed_range_readers() {
11200            return crate::engine::graph::StructuralOccupancy::default();
11201        }
11202        let mut occupancy = self.graph.structural_occupancy(sheet_id);
11203        if let Some(arrow_sheet) = self.arrow_sheets.sheet(sheet) {
11204            occupancy.include_arrow_sheet(arrow_sheet);
11205            occupancy
11206        } else {
11207            // Missing Arrow state cannot prove an apparently empty column empty.
11208            crate::engine::graph::StructuralOccupancy::conservative()
11209        }
11210    }
11211
11212    fn structural_column_occupancy(&self) -> crate::engine::graph::StructuralOccupancy {
11213        // Arrow exposes occupied columns through chunk metadata and overlay maps,
11214        // but has no cheap occupied-row index. Column edits therefore deliberately
11215        // retain conservative cross-axis invalidation instead of scanning cells.
11216        crate::engine::graph::StructuralOccupancy::conservative()
11217    }
11218
11219    /// Insert rows (1-based) and mirror into Arrow store when enabled
11220    pub fn insert_rows(
11221        &mut self,
11222        sheet: &str,
11223        before: u32,
11224        count: u32,
11225    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11226    {
11227        if count == 0 {
11228            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11229        }
11230        self.observe_function_semantic_epoch()
11231            .map_err(crate::engine::EditorError::Excel)?;
11232        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11233        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11234        let sheet_id = self.ensure_known_sheet_id(sheet)?;
11235        let before0 = before.saturating_sub(1);
11236        self.check_insert_room(sheet, before0, count, true)?;
11237        let affected_region = Self::structural_row_region(sheet_id, before0);
11238        let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11239        let summary = {
11240            let mut editor =
11241                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11242            editor.insert_rows(sheet_id, before0, count)?
11243        };
11244        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11245            let before0 = before0 as usize;
11246            asheet.insert_rows(before0, count as usize);
11247        }
11248        self.purge_derived_formats_after_row(sheet_id, before0);
11249        self.mark_moved_formula_vertices_dirty(&summary);
11250        self.clear_computed_overlay_after_row(sheet, before0 as usize);
11251        self.shift_row_visibility_insert(sheet_id, before0, count);
11252        self.record_structural_change(StructuralScope::Region(affected_region));
11253        self.mark_topology_edited();
11254        Ok(summary)
11255    }
11256
11257    /// Delete rows (1-based) and mirror into Arrow store when enabled
11258    pub fn delete_rows(
11259        &mut self,
11260        sheet: &str,
11261        start: u32,
11262        count: u32,
11263    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11264    {
11265        if count == 0 {
11266            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11267        }
11268        self.observe_function_semantic_epoch()
11269            .map_err(crate::engine::EditorError::Excel)?;
11270        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11271        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11272        let sheet_id = self.ensure_known_sheet_id(sheet)?;
11273        let start0 = start.saturating_sub(1);
11274        let affected_region = Self::structural_row_region(sheet_id, start0);
11275        let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11276        let summary = {
11277            let mut editor =
11278                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11279            editor.delete_rows(sheet_id, start0, count)?
11280        };
11281        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11282            let start0 = start0 as usize;
11283            asheet.delete_rows(start0, count as usize);
11284        }
11285        self.purge_derived_formats_after_row(sheet_id, start0);
11286        self.mark_moved_formula_vertices_dirty(&summary);
11287        self.clear_computed_overlay_after_row(sheet, start0 as usize);
11288        self.shift_row_visibility_delete(sheet_id, start0, count);
11289        self.record_structural_change(StructuralScope::Region(affected_region));
11290        self.mark_topology_edited();
11291        Ok(summary)
11292    }
11293
11294    /// Insert columns (1-based) and mirror into Arrow store when enabled
11295    pub fn insert_columns(
11296        &mut self,
11297        sheet: &str,
11298        before: u32,
11299        count: u32,
11300    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11301    {
11302        if count == 0 {
11303            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11304        }
11305        self.observe_function_semantic_epoch()
11306            .map_err(crate::engine::EditorError::Excel)?;
11307        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11308        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11309        let sheet_id = self.graph.sheet_id(sheet).ok_or(
11310            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11311                name: sheet.to_string(),
11312                reason: "Unknown sheet".to_string(),
11313            },
11314        )?;
11315        let before0 = before.saturating_sub(1);
11316        self.check_insert_room(sheet, before0, count, false)?;
11317        let affected_region = Self::structural_col_region(sheet_id, before0);
11318        let occupancy = self.structural_column_occupancy();
11319        let summary = {
11320            let mut editor =
11321                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11322            editor.insert_columns(sheet_id, before0, count)?
11323        };
11324        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11325            let before0 = before0 as usize;
11326            asheet.insert_columns(before0, count as usize);
11327        }
11328        self.purge_derived_formats_after_col(sheet_id, before0);
11329        self.mark_moved_formula_vertices_dirty(&summary);
11330        self.clear_computed_overlay_after_col(sheet, before0 as usize);
11331        self.record_structural_change(StructuralScope::Region(affected_region));
11332        self.mark_topology_edited();
11333        Ok(summary)
11334    }
11335
11336    /// Delete columns (1-based) and mirror into Arrow store when enabled
11337    pub fn delete_columns(
11338        &mut self,
11339        sheet: &str,
11340        start: u32,
11341        count: u32,
11342    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11343    {
11344        if count == 0 {
11345            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11346        }
11347        self.observe_function_semantic_epoch()
11348            .map_err(crate::engine::EditorError::Excel)?;
11349        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11350        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11351        let sheet_id = self.graph.sheet_id(sheet).ok_or(
11352            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11353                name: sheet.to_string(),
11354                reason: "Unknown sheet".to_string(),
11355            },
11356        )?;
11357        let start0 = start.saturating_sub(1);
11358        let affected_region = Self::structural_col_region(sheet_id, start0);
11359        let occupancy = self.structural_column_occupancy();
11360        let summary = {
11361            let mut editor =
11362                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11363            editor.delete_columns(sheet_id, start0, count)?
11364        };
11365        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11366            let start0 = start0 as usize;
11367            asheet.delete_columns(start0, count as usize);
11368        }
11369        self.purge_derived_formats_after_col(sheet_id, start0);
11370        self.mark_moved_formula_vertices_dirty(&summary);
11371        self.clear_computed_overlay_after_col(sheet, start0 as usize);
11372        self.record_structural_change(StructuralScope::Region(affected_region));
11373        self.mark_topology_edited();
11374        Ok(summary)
11375    }
11376    /// Arrow-backed used row bounds across a column span (1-based inclusive cols).
11377    fn arrow_used_row_bounds(
11378        &self,
11379        sheet: &str,
11380        start_col: u32,
11381        end_col: u32,
11382    ) -> Option<(u32, u32)> {
11383        let a = self.sheet_store().sheet(sheet)?;
11384        if a.columns.is_empty() {
11385            return None;
11386        }
11387        let sc0 = start_col.saturating_sub(1) as usize;
11388        let ec0 = end_col.saturating_sub(1) as usize;
11389        let col_hi = a.columns.len().saturating_sub(1);
11390        if sc0 > col_hi {
11391            return None;
11392        }
11393        let ec0 = ec0.min(col_hi);
11394        // Pass-scoped cache with snapshot guard
11395        let snap = self.data_snapshot_id();
11396        let mut min_r0: Option<usize> = None;
11397        for ci in sc0..=ec0 {
11398            let sheet_id = self.graph.sheet_id(sheet)?;
11399            if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
11400                g.as_ref()
11401                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11402            }) {
11403                let mv = mv as usize;
11404                min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
11405                continue;
11406            }
11407            // Compute and store
11408            let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11409            if let Ok(mut g) = self.row_bounds_cache.write() {
11410                g.get_or_insert_with(|| RowBoundsCache::new(snap))
11411                    .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
11412            }
11413            if let Some(m) = min_c {
11414                min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
11415            }
11416        }
11417        min_r0?;
11418        let mut max_r0: Option<usize> = None;
11419        for ci in sc0..=ec0 {
11420            let sheet_id = self.graph.sheet_id(sheet)?;
11421            if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
11422                g.as_ref()
11423                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11424            }) {
11425                let mv = mv as usize;
11426                max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
11427                continue;
11428            }
11429            let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11430            if let Ok(mut g) = self.row_bounds_cache.write() {
11431                g.get_or_insert_with(|| RowBoundsCache::new(snap))
11432                    .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
11433            }
11434            if let Some(m) = max_c {
11435                max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
11436            }
11437        }
11438        match (min_r0, max_r0) {
11439            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11440            _ => None,
11441        }
11442    }
11443
11444    fn scan_column_used_bounds(
11445        a: &crate::arrow_store::ArrowSheet,
11446        ci: usize,
11447    ) -> (Option<u32>, Option<u32>) {
11448        let col = &a.columns[ci];
11449
11450        // Min: scan dense chunks first, then sparse chunks in ascending index order.
11451        let mut min_r0: Option<u32> = None;
11452        for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11453            let tags = chunk.type_tag.values();
11454            for (off, &t) in tags.iter().enumerate() {
11455                let overlay_non_empty = chunk
11456                    .overlay
11457                    .get(off)
11458                    .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11459                    .unwrap_or(false)
11460                    || chunk
11461                        .computed_overlay
11462                        .get(off)
11463                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11464                        .unwrap_or(false);
11465                if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11466                    let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11467                        break;
11468                    };
11469                    let row0 = chunk_start + off;
11470                    min_r0 = Some(row0 as u32);
11471                    break;
11472                }
11473            }
11474            if min_r0.is_some() {
11475                break;
11476            }
11477        }
11478        if min_r0.is_none() && !col.sparse_chunks.is_empty() {
11479            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11480            sparse_idxs.sort_unstable();
11481            for chunk_idx in sparse_idxs {
11482                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11483                    continue;
11484                };
11485                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11486                    continue;
11487                };
11488                let tags = chunk.type_tag.values();
11489                for (off, &t) in tags.iter().enumerate() {
11490                    let overlay_non_empty = chunk
11491                        .overlay
11492                        .get(off)
11493                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11494                        .unwrap_or(false)
11495                        || chunk
11496                            .computed_overlay
11497                            .get(off)
11498                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11499                            .unwrap_or(false);
11500                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11501                        let row0 = chunk_start + off;
11502                        min_r0 = Some(row0 as u32);
11503                        break;
11504                    }
11505                }
11506                if min_r0.is_some() {
11507                    break;
11508                }
11509            }
11510        }
11511
11512        // Max: scan sparse chunks in descending index order, then dense chunks in reverse.
11513        let mut max_r0: Option<u32> = None;
11514        if !col.sparse_chunks.is_empty() {
11515            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11516            sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
11517            for chunk_idx in sparse_idxs {
11518                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11519                    continue;
11520                };
11521                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11522                    continue;
11523                };
11524                let tags = chunk.type_tag.values();
11525                for (rev_idx, &t) in tags.iter().enumerate().rev() {
11526                    let overlay_non_empty = chunk
11527                        .overlay
11528                        .get(rev_idx)
11529                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11530                        .unwrap_or(false)
11531                        || chunk
11532                            .computed_overlay
11533                            .get(rev_idx)
11534                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11535                            .unwrap_or(false);
11536                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11537                        let row0 = chunk_start + rev_idx;
11538                        max_r0 = Some(row0 as u32);
11539                        break;
11540                    }
11541                }
11542                if max_r0.is_some() {
11543                    break;
11544                }
11545            }
11546        }
11547        if max_r0.is_none() {
11548            for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
11549                let tags = chunk.type_tag.values();
11550                for (rev_idx, &t) in tags.iter().enumerate().rev() {
11551                    let overlay_non_empty = chunk
11552                        .overlay
11553                        .get(rev_idx)
11554                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11555                        .unwrap_or(false)
11556                        || chunk
11557                            .computed_overlay
11558                            .get(rev_idx)
11559                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11560                            .unwrap_or(false);
11561                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11562                        let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11563                            break;
11564                        };
11565                        let row0 = chunk_start + rev_idx;
11566                        max_r0 = Some(row0 as u32);
11567                        break;
11568                    }
11569                }
11570                if max_r0.is_some() {
11571                    break;
11572                }
11573            }
11574        }
11575
11576        (min_r0, max_r0)
11577    }
11578
11579    /// Arrow-backed used column bounds across a row span (1-based inclusive rows).
11580    fn arrow_used_col_bounds(
11581        &self,
11582        sheet: &str,
11583        start_row: u32,
11584        end_row: u32,
11585    ) -> Option<(u32, u32)> {
11586        let a = self.sheet_store().sheet(sheet)?;
11587        if a.columns.is_empty() {
11588            return None;
11589        }
11590        let sr0 = start_row.saturating_sub(1) as usize;
11591        let er0 = end_row.saturating_sub(1) as usize;
11592        if sr0 > er0 {
11593            return None;
11594        }
11595        // Map start/end rows into chunk ranges
11596        // We will scan each column for any non-empty within [sr0..=er0]
11597        let mut min_c0: Option<usize> = None;
11598        let mut max_c0: Option<usize> = None;
11599        // Precompute chunk bounds for row range
11600        for (ci, col) in a.columns.iter().enumerate() {
11601            let mut any_in_range = false;
11602
11603            let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
11604                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11605                    return false;
11606                };
11607                let chunk_len = chunk.type_tag.len();
11608                if chunk_len == 0 {
11609                    return false;
11610                }
11611                let chunk_end = chunk_start + chunk_len.saturating_sub(1);
11612                // check intersection
11613                if sr0 > chunk_end || er0 < chunk_start {
11614                    return false;
11615                }
11616                let start_off = sr0.max(chunk_start) - chunk_start;
11617                let end_off = er0.min(chunk_end) - chunk_start;
11618                let tags = chunk.type_tag.values();
11619                for off in start_off..=end_off {
11620                    let overlay_non_empty = chunk
11621                        .overlay
11622                        .get(off)
11623                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11624                        .unwrap_or(false)
11625                        || chunk
11626                            .computed_overlay
11627                            .get(off)
11628                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11629                            .unwrap_or(false);
11630                    if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
11631                        return true;
11632                    }
11633                }
11634                false
11635            };
11636
11637            for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11638                if scan_chunk(chunk_idx, chunk) {
11639                    any_in_range = true;
11640                    break;
11641                }
11642            }
11643
11644            if !any_in_range && !col.sparse_chunks.is_empty() {
11645                for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
11646                    if scan_chunk(chunk_idx, chunk) {
11647                        any_in_range = true;
11648                        break;
11649                    }
11650                }
11651            }
11652
11653            if any_in_range {
11654                min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
11655                max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
11656            }
11657        }
11658        match (min_c0, max_c0) {
11659            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11660            _ => None,
11661        }
11662    }
11663
11664    fn formula_row_bounds_for_columns(
11665        &self,
11666        sheet: &str,
11667        start_col: u32,
11668        end_col: u32,
11669    ) -> Option<(u32, u32)> {
11670        let sheet_id = self.graph.sheet_id(sheet)?;
11671        let sc0 = start_col.saturating_sub(1);
11672        let ec0 = end_col.saturating_sub(1);
11673        let mut min_r0: Option<u32> = None;
11674        let mut max_r0: Option<u32> = None;
11675
11676        if self.graph.sheet_index(sheet_id).is_some() {
11677            for vid in self.graph.vertices_in_cols(sheet_id, sc0, ec0) {
11678                if !matches!(
11679                    self.graph.get_vertex_kind(vid),
11680                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11681                ) {
11682                    continue;
11683                }
11684                let Some(row0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.row()) else {
11685                    continue;
11686                };
11687                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11688                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11689            }
11690        } else {
11691            for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11692                if !matches!(
11693                    self.graph.get_vertex_kind(vid),
11694                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11695                ) {
11696                    continue;
11697                }
11698                let col0 = coord.col();
11699                if col0 < sc0 || col0 > ec0 {
11700                    continue;
11701                }
11702                let row0 = coord.row();
11703                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11704                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11705            }
11706        }
11707
11708        match (min_r0, max_r0) {
11709            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11710            _ => None,
11711        }
11712    }
11713
11714    fn formula_col_bounds_for_rows(
11715        &self,
11716        sheet: &str,
11717        start_row: u32,
11718        end_row: u32,
11719    ) -> Option<(u32, u32)> {
11720        let sheet_id = self.graph.sheet_id(sheet)?;
11721        let sr0 = start_row.saturating_sub(1);
11722        let er0 = end_row.saturating_sub(1);
11723        let mut min_c0: Option<u32> = None;
11724        let mut max_c0: Option<u32> = None;
11725
11726        if self.graph.sheet_index(sheet_id).is_some() {
11727            for vid in self.graph.vertices_in_rows(sheet_id, sr0, er0) {
11728                if !matches!(
11729                    self.graph.get_vertex_kind(vid),
11730                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11731                ) {
11732                    continue;
11733                }
11734                let Some(col0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.col()) else {
11735                    continue;
11736                };
11737                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11738                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11739            }
11740        } else {
11741            for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11742                if !matches!(
11743                    self.graph.get_vertex_kind(vid),
11744                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11745                ) {
11746                    continue;
11747                }
11748                let row0 = coord.row();
11749                if row0 < sr0 || row0 > er0 {
11750                    continue;
11751                }
11752                let col0 = coord.col();
11753                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11754                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11755            }
11756        }
11757
11758        match (min_c0, max_c0) {
11759            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11760            _ => None,
11761        }
11762    }
11763
11764    fn union_used_bounds(
11765        first: Option<(u32, u32)>,
11766        second: Option<(u32, u32)>,
11767    ) -> Option<(u32, u32)> {
11768        match (first, second) {
11769            (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
11770            (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
11771            (None, None) => None,
11772        }
11773    }
11774
11775    /// Mirror a single cell value into the Arrow overlay if enabled.
11776    /// Handles capacity growth, per-chunk overlay set, and heuristic compaction.
11777    fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
11778        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11779            return;
11780        }
11781        if self.arrow_sheets.sheet(sheet).is_none() {
11782            self.arrow_sheets
11783                .sheets
11784                .push(crate::arrow_store::ArrowSheet {
11785                    name: std::sync::Arc::<str>::from(sheet),
11786                    date_system: self.config.date_system,
11787                    columns: Vec::new(),
11788                    nrows: 0,
11789                    chunk_starts: Vec::new(),
11790                    chunk_rows: 32 * 1024,
11791                });
11792        }
11793
11794        let row0 = row.saturating_sub(1) as usize;
11795        let col0 = col.saturating_sub(1) as usize;
11796
11797        let asheet = self
11798            .arrow_sheets
11799            .sheet_mut(sheet)
11800            .expect("ArrowSheet must exist");
11801
11802        let cur_cols = asheet.columns.len();
11803        if col0 >= cur_cols {
11804            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11805        }
11806
11807        if row0 >= asheet.nrows as usize {
11808            if asheet.columns.is_empty() {
11809                asheet.insert_columns(0, 1);
11810            }
11811            asheet.ensure_row_capacity(row0 + 1);
11812        }
11813        if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
11814            let ov =
11815                crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
11816            let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
11817                let _ = ch.overlay.set(in_off, ov);
11818                let format = match value {
11819                    LiteralValue::Date(_) => Some(crate::format::FormatId::DATE),
11820                    LiteralValue::DateTime(_) => Some(crate::format::FormatId::DATETIME),
11821                    LiteralValue::Time(_) => Some(crate::format::FormatId::TIME),
11822                    LiteralValue::Duration(_) => Some(crate::format::FormatId::DURATION),
11823                    _ => None,
11824                };
11825                ch.overlay.set_format(in_off, format);
11826                // A user edit must invalidate any computed (formula/spill) overlay entry at
11827                // this cell. Otherwise, if the delta overlay later compacts into the base lanes
11828                // (clearing `overlay`), a stale `computed_overlay=Empty` could incorrectly mask
11829                // the edited base value under the read cascade.
11830                ch.computed_overlay.remove(in_off)
11831            } else {
11832                return;
11833            };
11834            // Heuristic compaction: > len/50 or > 1024
11835            let abs_threshold = 1024usize;
11836            let frac_den = 50usize;
11837            let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
11838            if freed > 0 {
11839                self.overlay_compactions = self.overlay_compactions.saturating_add(1);
11840            }
11841            self.adjust_computed_overlay_bytes(computed_delta);
11842        }
11843    }
11844
11845    /// Remove a delta-overlay entry for a single cell (if present).
11846    ///
11847    /// This is used when transitioning a cell to a formula so that any previous user-edit overlay
11848    /// does not continue to mask computed overlay outputs.
11849    fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
11850        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11851            return;
11852        }
11853        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11854            return;
11855        };
11856        let row0 = row.saturating_sub(1) as usize;
11857        let col0 = col.saturating_sub(1) as usize;
11858        if row0 >= asheet.nrows as usize {
11859            return;
11860        }
11861        if col0 >= asheet.columns.len() {
11862            return;
11863        }
11864        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11865            return;
11866        };
11867        if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
11868            let _ = ch.overlay.remove(in_off);
11869        }
11870    }
11871
11872    fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
11873        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11874            return;
11875        }
11876
11877        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11878            return;
11879        };
11880        if start_row0 >= asheet.nrows as usize {
11881            return;
11882        }
11883
11884        let starts = asheet.chunk_starts.clone();
11885        let nrows = asheet.nrows as usize;
11886        let mut delta = 0isize;
11887        for col in &mut asheet.columns {
11888            for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
11889                let Some(&chunk_start) = starts.get(chunk_idx) else {
11890                    continue;
11891                };
11892                let chunk_end = starts
11893                    .get(chunk_idx + 1)
11894                    .copied()
11895                    .unwrap_or(nrows)
11896                    .min(chunk_start.saturating_add(ch.len()));
11897                if chunk_end <= start_row0 {
11898                    continue;
11899                }
11900                if chunk_start >= start_row0 {
11901                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11902                } else {
11903                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11904                    delta = delta
11905                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11906                }
11907            }
11908
11909            for (chunk_idx, ch) in &mut col.sparse_chunks {
11910                let Some(&chunk_start) = starts.get(*chunk_idx) else {
11911                    continue;
11912                };
11913                let chunk_end = starts
11914                    .get(*chunk_idx + 1)
11915                    .copied()
11916                    .unwrap_or(nrows)
11917                    .min(chunk_start.saturating_add(ch.len()));
11918                if chunk_end <= start_row0 {
11919                    continue;
11920                }
11921                if chunk_start >= start_row0 {
11922                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11923                } else {
11924                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11925                    delta = delta
11926                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11927                }
11928            }
11929        }
11930        self.adjust_computed_overlay_bytes(delta);
11931    }
11932
11933    fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
11934        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11935            return;
11936        }
11937
11938        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11939            return;
11940        };
11941        if start_col0 >= asheet.columns.len() {
11942            return;
11943        }
11944
11945        let mut delta = 0isize;
11946        for col in asheet.columns.iter_mut().skip(start_col0) {
11947            for ch in &mut col.chunks {
11948                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11949            }
11950            for ch in col.sparse_chunks.values_mut() {
11951                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11952            }
11953        }
11954        self.adjust_computed_overlay_bytes(delta);
11955    }
11956
11957    #[inline]
11958    fn literal_to_overlay_value(
11959        value: &LiteralValue,
11960        date_system: crate::engine::DateSystem,
11961    ) -> crate::arrow_store::OverlayValue {
11962        crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
11963    }
11964
11965    fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
11966        self.arrow_sheets
11967            .sheet(sheet)
11968            .map(|sheet| sheet.date_system)
11969            .unwrap_or(self.config.date_system)
11970    }
11971
11972    /// Read a single cell's delta overlay entry (if present), preserving the distinction between
11973    /// absent and explicit `Empty`.
11974    fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11975        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11976            return None;
11977        }
11978        let asheet = self.arrow_sheets.sheet(sheet)?;
11979        let row0 = row.saturating_sub(1) as usize;
11980        let col0 = col.saturating_sub(1) as usize;
11981        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11982            return None;
11983        }
11984        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
11985        let ch = asheet.columns[col0].chunk(ch_idx)?;
11986        ch.overlay
11987            .get_scalar(in_off)
11988            .map(|ov| ov.to_literal_for(asheet.date_system))
11989    }
11990
11991    /// Read a single cell's computed overlay entry (if present), preserving the distinction
11992    /// between absent and explicit `Empty`.
11993    fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11994        if !(self.config.arrow_storage_enabled
11995            && self.config.delta_overlay_enabled
11996            && self.config.write_formula_overlay_enabled)
11997        {
11998            return None;
11999        }
12000        let asheet = self.arrow_sheets.sheet(sheet)?;
12001        let row0 = row.saturating_sub(1) as usize;
12002        let col0 = col.saturating_sub(1) as usize;
12003        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
12004            return None;
12005        }
12006        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
12007        let ch = asheet.columns[col0].chunk(ch_idx)?;
12008        ch.computed_overlay
12009            .get_scalar(in_off)
12010            .map(|ov| ov.to_literal_for(asheet.date_system))
12011    }
12012
12013    fn set_delta_overlay_cell_raw(
12014        &mut self,
12015        sheet: &str,
12016        row: u32,
12017        col: u32,
12018        value: Option<LiteralValue>,
12019    ) {
12020        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
12021            return;
12022        }
12023
12024        self.ensure_arrow_sheet(sheet);
12025        let date_system = self.arrow_sheet_date_system(sheet);
12026        let ov_opt = value
12027            .as_ref()
12028            .map(|value| Self::literal_to_overlay_value(value, date_system));
12029        let row0 = row.saturating_sub(1) as usize;
12030        let col0 = col.saturating_sub(1) as usize;
12031        let asheet = self
12032            .arrow_sheets
12033            .sheet_mut(sheet)
12034            .expect("ArrowSheet must exist");
12035
12036        let cur_cols = asheet.columns.len();
12037        if col0 >= cur_cols {
12038            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12039        }
12040        if row0 >= asheet.nrows as usize {
12041            if asheet.columns.is_empty() {
12042                asheet.insert_columns(0, 1);
12043            }
12044            asheet.ensure_row_capacity(row0 + 1);
12045        }
12046
12047        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12048            return;
12049        };
12050        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12051            return;
12052        };
12053
12054        if let Some(ov) = ov_opt {
12055            let _ = ch.overlay.set(in_off, ov);
12056        } else {
12057            let _ = ch.overlay.remove(in_off);
12058        }
12059    }
12060
12061    fn set_computed_overlay_cell_raw(
12062        &mut self,
12063        sheet: &str,
12064        row: u32,
12065        col: u32,
12066        value: Option<LiteralValue>,
12067    ) {
12068        if !(self.config.arrow_storage_enabled
12069            && self.config.delta_overlay_enabled
12070            && self.config.write_formula_overlay_enabled)
12071        {
12072            return;
12073        }
12074
12075        self.ensure_arrow_sheet(sheet);
12076        let date_system = self.arrow_sheet_date_system(sheet);
12077        let ov_opt = value
12078            .as_ref()
12079            .map(|value| Self::literal_to_overlay_value(value, date_system));
12080        let row0 = row.saturating_sub(1) as usize;
12081        let col0 = col.saturating_sub(1) as usize;
12082        let asheet = self
12083            .arrow_sheets
12084            .sheet_mut(sheet)
12085            .expect("ArrowSheet must exist");
12086
12087        let cur_cols = asheet.columns.len();
12088        if col0 >= cur_cols {
12089            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12090        }
12091        if row0 >= asheet.nrows as usize {
12092            if asheet.columns.is_empty() {
12093                asheet.insert_columns(0, 1);
12094            }
12095            asheet.ensure_row_capacity(row0 + 1);
12096        }
12097
12098        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12099            return;
12100        };
12101        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12102            return;
12103        };
12104
12105        let delta = if let Some(ov) = ov_opt {
12106            ch.computed_overlay.set(in_off, ov)
12107        } else {
12108            ch.computed_overlay.remove(in_off)
12109        };
12110        self.adjust_computed_overlay_bytes(delta);
12111    }
12112
12113    fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
12114        use crate::engine::ArrowOp;
12115
12116        let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
12117            Box::new(batch.ops.iter().rev())
12118        } else {
12119            Box::new(batch.ops.iter())
12120        };
12121
12122        for op in iter {
12123            match op {
12124                ArrowOp::SetDeltaCell {
12125                    sheet_id,
12126                    row0,
12127                    col0,
12128                    old,
12129                    new,
12130                } => {
12131                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12132                    let v = if undo { old.clone() } else { new.clone() };
12133                    self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
12134                }
12135                ArrowOp::SetComputedCell {
12136                    sheet_id,
12137                    row0,
12138                    col0,
12139                    old,
12140                    new,
12141                } => {
12142                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12143                    let v = if undo { old.clone() } else { new.clone() };
12144                    self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
12145                }
12146                ArrowOp::RestoreComputedRect {
12147                    sheet_id,
12148                    sr0,
12149                    sc0,
12150                    er0,
12151                    ec0,
12152                    old,
12153                    new,
12154                } => {
12155                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12156                    let vals = if undo { old } else { new };
12157                    let height = (*er0).saturating_sub(*sr0) as usize + 1;
12158                    let width = (*ec0).saturating_sub(*sc0) as usize + 1;
12159                    for r in 0..height {
12160                        for c in 0..width {
12161                            let v = vals
12162                                .get(r)
12163                                .and_then(|row| row.get(c))
12164                                .cloned()
12165                                .unwrap_or(LiteralValue::Empty);
12166                            self.set_computed_overlay_cell_raw(
12167                                &sheet,
12168                                *sr0 + 1 + r as u32,
12169                                *sc0 + 1 + c as u32,
12170                                Some(v),
12171                            );
12172                        }
12173                    }
12174                }
12175                ArrowOp::InsertRows {
12176                    sheet_id,
12177                    before0,
12178                    count,
12179                } => {
12180                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12181                    self.ensure_arrow_sheet(&sheet);
12182                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
12183                        if undo {
12184                            asheet.delete_rows(*before0 as usize, *count as usize);
12185                        } else {
12186                            asheet.insert_rows(*before0 as usize, *count as usize);
12187                        }
12188                    }
12189                    self.purge_derived_formats_after_row(*sheet_id, *before0);
12190                }
12191                ArrowOp::InsertCols {
12192                    sheet_id,
12193                    before0,
12194                    count,
12195                } => {
12196                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
12197                    self.ensure_arrow_sheet(&sheet);
12198                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
12199                        if undo {
12200                            asheet.delete_columns(*before0 as usize, *count as usize);
12201                        } else {
12202                            asheet.insert_columns(*before0 as usize, *count as usize);
12203                        }
12204                    }
12205                    self.purge_derived_formats_after_col(*sheet_id, *before0);
12206                }
12207            }
12208        }
12209    }
12210
12211    fn record_spill_ops_into_arrow_undo(
12212        &mut self,
12213        undo: &mut crate::engine::ArrowUndoBatch,
12214        events: &[crate::engine::ChangeEvent],
12215    ) {
12216        use crate::engine::ChangeEvent;
12217        use formualizer_common::LiteralValue;
12218
12219        #[allow(clippy::type_complexity)]
12220        let rect_from_snapshot =
12221            |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
12222             -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
12223                if snap.target_cells.is_empty() {
12224                    return None;
12225                }
12226                let sheet_id = snap.target_cells[0].sheet_id;
12227                let sr0 = snap.target_cells[0].coord.row();
12228                let sc0 = snap.target_cells[0].coord.col();
12229                if snap.values.is_empty() || snap.values[0].is_empty() {
12230                    return None;
12231                }
12232                let h = snap.values.len() as u32;
12233                let w = snap.values[0].len() as u32;
12234                let er0 = sr0.saturating_add(h.saturating_sub(1));
12235                let ec0 = sc0.saturating_add(w.saturating_sub(1));
12236                Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
12237            };
12238
12239        for ev in events {
12240            match ev {
12241                ChangeEvent::SpillCommitted { old, new, .. } => {
12242                    if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
12243                        let old_vals = if let Some(old_snap) = old {
12244                            rect_from_snapshot(old_snap)
12245                                .map(|(_, _, _, _, _, v)| v)
12246                                .unwrap_or_else(|| {
12247                                    vec![
12248                                        vec![LiteralValue::Empty; new_vals[0].len()];
12249                                        new_vals.len()
12250                                    ]
12251                                })
12252                        } else {
12253                            vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
12254                        };
12255                        undo.record_restore_computed_rect(
12256                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
12257                        );
12258                    }
12259                }
12260                ChangeEvent::SpillCleared { old, .. } => {
12261                    if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
12262                        let new_vals =
12263                            vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
12264                        undo.record_restore_computed_rect(
12265                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
12266                        );
12267                    }
12268                }
12269                _ => {}
12270            }
12271        }
12272    }
12273
12274    /// Mirror a value into the computed overlay (formula/spill outputs).
12275    ///
12276    /// This path is subject to `EvalConfig.max_overlay_memory_bytes`.
12277    /// If the cap is exceeded, computed overlays are compacted into base lanes.
12278    fn mirror_value_to_computed_overlay(
12279        &mut self,
12280        sheet: &str,
12281        row: u32,
12282        col: u32,
12283        value: &LiteralValue,
12284    ) {
12285        if !(self.config.arrow_storage_enabled
12286            && self.config.delta_overlay_enabled
12287            && self.config.write_formula_overlay_enabled)
12288        {
12289            return;
12290        }
12291        if self.computed_overlay_mirroring_disabled {
12292            return;
12293        }
12294
12295        let date_system = self.arrow_sheet_date_system(sheet);
12296        let ov = Self::literal_to_overlay_value(value, date_system);
12297        self.write_computed_overlay_value_0based(
12298            sheet,
12299            row.saturating_sub(1),
12300            col.saturating_sub(1),
12301            ov,
12302        );
12303    }
12304
12305    fn record_derived_format(&self, vertex_id: VertexId, format: Option<crate::format::FormatId>) {
12306        if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
12307            self.record_derived_format_at(cell, format);
12308        }
12309    }
12310
12311    fn record_derived_format_at(&self, cell: CellRef, format: Option<crate::format::FormatId>) {
12312        #[cfg(test)]
12313        self.derived_format_operations_for_test
12314            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12315        let format = format.filter(|id| *id != crate::format::FormatId::GENERAL);
12316        self.derived_formats.set(cell, format);
12317    }
12318
12319    fn clear_cell_format_state(&mut self, sheet: &str, cell: CellRef) {
12320        self.derived_formats.set(cell, None);
12321        if let Some(arrow) = self.arrow_sheets.sheet_mut(sheet) {
12322            arrow.clear_format(cell.coord.row() as usize, cell.coord.col() as usize);
12323        }
12324    }
12325
12326    fn clear_logged_cell_format_states(&mut self, events: &[ChangeEvent]) {
12327        let cells = events
12328            .iter()
12329            .filter_map(|event| match event {
12330                ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
12331                    Some(*addr)
12332                }
12333                _ => None,
12334            })
12335            .collect::<FxHashSet<_>>();
12336        for cell in cells {
12337            let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12338            self.clear_cell_format_state(&sheet, cell);
12339        }
12340    }
12341
12342    fn purge_derived_formats_after_row(&mut self, sheet_id: SheetId, start0: u32) {
12343        self.derived_formats
12344            .retain(|cell| cell.sheet_id != sheet_id || cell.coord.row() < start0);
12345    }
12346
12347    fn purge_derived_formats_after_col(&mut self, sheet_id: SheetId, start0: u32) {
12348        self.derived_formats
12349            .retain(|cell| cell.sheet_id != sheet_id || cell.coord.col() < start0);
12350    }
12351
12352    fn purge_derived_formats_for_sheet(&mut self, sheet_id: SheetId) {
12353        self.derived_formats
12354            .retain(|cell| cell.sheet_id != sheet_id);
12355    }
12356
12357    #[cfg(test)]
12358    pub(crate) fn debug_computed_overlay_format_0based(
12359        &self,
12360        sheet: &str,
12361        row0: u32,
12362        col0: u32,
12363    ) -> Option<crate::format::FormatId> {
12364        let sheet = self.arrow_sheets.sheet(sheet)?;
12365        let (chunk_idx, row_in_chunk) = sheet.chunk_of_row(row0 as usize)?;
12366        sheet
12367            .columns
12368            .get(col0 as usize)?
12369            .chunk(chunk_idx)?
12370            .computed_overlay
12371            .get_format(row_in_chunk)
12372    }
12373
12374    #[cfg(test)]
12375    pub(crate) fn debug_computed_overlay_chunk_has_formats_0based(
12376        &self,
12377        sheet: &str,
12378        row0: u32,
12379        col0: u32,
12380    ) -> bool {
12381        let Some(sheet) = self.arrow_sheets.sheet(sheet) else {
12382            return false;
12383        };
12384        let Some((chunk_idx, _)) = sheet.chunk_of_row(row0 as usize) else {
12385            return false;
12386        };
12387        sheet
12388            .columns
12389            .get(col0 as usize)
12390            .and_then(|column| column.chunk(chunk_idx))
12391            .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12392    }
12393
12394    #[cfg(test)]
12395    pub(crate) fn debug_clear_derived_format_0based(&mut self, sheet: &str, row0: u32, col0: u32) {
12396        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12397            self.derived_formats
12398                .set(CellRef::new_absolute(sheet_id, row0, col0), None);
12399        }
12400    }
12401
12402    #[cfg(test)]
12403    pub(crate) fn debug_record_derived_format_0based(
12404        &self,
12405        sheet: &str,
12406        row0: u32,
12407        col0: u32,
12408        format: Option<crate::format::FormatId>,
12409    ) {
12410        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12411            self.record_derived_format_at(CellRef::new_absolute(sheet_id, row0, col0), format);
12412        }
12413    }
12414
12415    #[cfg(test)]
12416    pub(crate) fn debug_derived_format_0based(
12417        &self,
12418        sheet: &str,
12419        row0: u32,
12420        col0: u32,
12421    ) -> Option<crate::format::FormatId> {
12422        let sheet_id = self.graph.sheet_id(sheet)?;
12423        self.derived_formats
12424            .get(&CellRef::new_absolute(sheet_id, row0, col0))
12425    }
12426
12427    #[cfg(test)]
12428    pub(crate) fn debug_reset_format_write_operation_counts(&mut self) {
12429        self.derived_format_operations_for_test
12430            .store(0, std::sync::atomic::Ordering::Relaxed);
12431        self.computed_overlay_set_explicit_entry_operations_for_test = 0;
12432        self.computed_overlay_stale_clear_range_effects_for_test = 0;
12433        self.computed_overlay_stale_clear_offset_attempts_for_test = 0;
12434        self.computed_format_vector_allocations_for_test
12435            .store(0, std::sync::atomic::Ordering::Relaxed);
12436    }
12437
12438    #[cfg(test)]
12439    pub(crate) fn debug_format_write_operation_counts(&self) -> (u64, u64, u64, u64, u64) {
12440        (
12441            self.derived_format_operations_for_test
12442                .load(std::sync::atomic::Ordering::Relaxed),
12443            self.computed_overlay_set_explicit_entry_operations_for_test,
12444            self.computed_format_vector_allocations_for_test
12445                .load(std::sync::atomic::Ordering::Relaxed),
12446            self.computed_overlay_stale_clear_range_effects_for_test,
12447            self.computed_overlay_stale_clear_offset_attempts_for_test,
12448        )
12449    }
12450
12451    fn write_computed_overlay_format_0based(
12452        &mut self,
12453        sheet: &str,
12454        row0: u32,
12455        col0: u32,
12456        format: Option<crate::format::FormatId>,
12457    ) {
12458        self.ensure_arrow_sheet(sheet);
12459        let (row0, col0) = (row0 as usize, col0 as usize);
12460        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
12461            return;
12462        };
12463        if col0 >= asheet.columns.len() {
12464            asheet.insert_columns(asheet.columns.len(), col0 + 1 - asheet.columns.len());
12465        }
12466        if row0 >= asheet.nrows as usize {
12467            asheet.ensure_row_capacity(row0 + 1);
12468        }
12469        let Some((chunk, offset)) = asheet.chunk_of_row(row0) else {
12470            return;
12471        };
12472        if let Some(chunk) = asheet.ensure_column_chunk_mut(col0, chunk) {
12473            chunk.computed_overlay.set_format(offset, format);
12474        }
12475    }
12476
12477    /// One unbuffered computed write of `value` and its derived `format` at
12478    /// `cell` (a single sheet lookup; the same writes as
12479    /// `write_computed_overlay_value_0based` then
12480    /// `write_computed_overlay_format_0based`).
12481    fn write_computed_cell_0based(
12482        &mut self,
12483        cell: CellRef,
12484        value: &LiteralValue,
12485        format: Option<crate::format::FormatId>,
12486    ) {
12487        if !(self.config.arrow_storage_enabled
12488            && self.config.delta_overlay_enabled
12489            && self.config.write_formula_overlay_enabled)
12490            || self.computed_overlay_mirroring_disabled
12491        {
12492            return;
12493        }
12494        if self.arrow_sheet_by_id(cell.sheet_id).is_none() {
12495            let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12496            self.ensure_arrow_sheet(&sheet);
12497        }
12498        let (row0, col0) = (cell.coord.row() as usize, cell.coord.col() as usize);
12499        let Some(asheet) = self.arrow_sheet_by_id_mut(cell.sheet_id) else {
12500            return;
12501        };
12502        let ov = Self::literal_to_overlay_value(value, asheet.date_system);
12503        let cur_cols = asheet.columns.len();
12504        if col0 >= cur_cols {
12505            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12506        }
12507        if row0 >= asheet.nrows as usize {
12508            asheet.ensure_row_capacity(row0 + 1);
12509        }
12510        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12511            return;
12512        };
12513        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12514            return;
12515        };
12516        let delta = ch.computed_overlay.set_scalar(in_off, ov);
12517        ch.computed_overlay.set_format(in_off, format);
12518        self.adjust_computed_overlay_bytes(delta);
12519        if let Some(cap) = self.config.max_overlay_memory_bytes
12520            && self.computed_overlay_bytes_estimate > cap
12521        {
12522            self.disable_computed_overlay_mirroring_due_to_budget(cap);
12523        }
12524    }
12525
12526    fn write_computed_overlay_value_0based(
12527        &mut self,
12528        sheet: &str,
12529        row0: u32,
12530        col0: u32,
12531        value: OverlayValue,
12532    ) {
12533        if !(self.config.arrow_storage_enabled
12534            && self.config.delta_overlay_enabled
12535            && self.config.write_formula_overlay_enabled)
12536        {
12537            return;
12538        }
12539        if self.computed_overlay_mirroring_disabled {
12540            return;
12541        }
12542
12543        self.ensure_arrow_sheet(sheet);
12544
12545        let row0 = row0 as usize;
12546        let col0 = col0 as usize;
12547        let asheet = self
12548            .arrow_sheets
12549            .sheet_mut(sheet)
12550            .expect("ArrowSheet must exist");
12551
12552        let cur_cols = asheet.columns.len();
12553        if col0 >= cur_cols {
12554            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12555        }
12556
12557        if row0 >= asheet.nrows as usize {
12558            if asheet.columns.is_empty() {
12559                asheet.insert_columns(0, 1);
12560            }
12561            asheet.ensure_row_capacity(row0 + 1);
12562        }
12563
12564        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12565            return;
12566        };
12567        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12568            return;
12569        };
12570
12571        let delta = ch.computed_overlay.set_scalar(in_off, value);
12572        self.adjust_computed_overlay_bytes(delta);
12573
12574        if let Some(cap) = self.config.max_overlay_memory_bytes
12575            && self.computed_overlay_bytes_estimate > cap
12576        {
12577            self.disable_computed_overlay_mirroring_due_to_budget(cap);
12578        }
12579    }
12580
12581    pub(crate) fn plan_computed_write_coalescing(
12582        &self,
12583        buffer: &ComputedWriteBuffer,
12584    ) -> ComputedWriteCoalescingPlan {
12585        self.plan_computed_write_coalescing_from_writes(
12586            buffer.writes().iter().cloned(),
12587            buffer.formats_present,
12588        )
12589    }
12590
12591    fn plan_owned_computed_write_coalescing(
12592        &self,
12593        writes: Vec<ComputedWrite>,
12594        formats_present: bool,
12595    ) -> ComputedWriteCoalescingPlan {
12596        self.plan_computed_write_coalescing_from_writes(writes, formats_present)
12597    }
12598
12599    fn plan_computed_write_coalescing_from_writes(
12600        &self,
12601        writes: impl IntoIterator<Item = ComputedWrite>,
12602        formats_present: bool,
12603    ) -> ComputedWriteCoalescingPlan {
12604        let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
12605            BTreeMap::new();
12606        let mut input_cells = 0usize;
12607        // One sheet lookup per sheet run of writes, not per cell.
12608        let mut located: Option<(SheetId, Option<&crate::arrow_store::ArrowSheet>)> = None;
12609
12610        for write in writes {
12611            match write {
12612                ComputedWrite::Cell {
12613                    seq,
12614                    sheet_id,
12615                    row0,
12616                    col0,
12617                    value,
12618                    format_id,
12619                } => {
12620                    input_cells = input_cells.saturating_add(1);
12621                    let sheet = match located {
12622                        Some((id, sheet)) if id == sheet_id => sheet,
12623                        _ => {
12624                            let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12625                            located = Some((sheet_id, sheet));
12626                            sheet
12627                        }
12628                    };
12629                    let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12630                        Some(sheet) => {
12631                            Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize)
12632                        }
12633                        None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12634                            row0 as usize,
12635                            32 * 1024,
12636                        ),
12637                    };
12638                    groups
12639                        .entry(ComputedWriteChunkKey {
12640                            sheet_id,
12641                            col0,
12642                            chunk_idx,
12643                            chunk_start_row0,
12644                        })
12645                        .or_default()
12646                        .push(ComputedWriteChunkEntryPlan {
12647                            row_in_chunk,
12648                            seq,
12649                            value,
12650                            format_id,
12651                        });
12652                }
12653                ComputedWrite::Run {
12654                    seq,
12655                    sheet_id,
12656                    row0,
12657                    col0,
12658                    entries,
12659                } => {
12660                    input_cells = input_cells.saturating_add(entries.len());
12661                    let sheet = match located {
12662                        Some((id, sheet)) if id == sheet_id => sheet,
12663                        _ => {
12664                            let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12665                            located = Some((sheet_id, sheet));
12666                            sheet
12667                        }
12668                    };
12669                    // Rows are located one by one (a binary search, no sheet
12670                    // lookup); the group map is touched once per chunk
12671                    // segment of the run.
12672                    let mut segment: Vec<ComputedWriteChunkEntryPlan> = Vec::new();
12673                    let mut segment_key: Option<ComputedWriteChunkKey> = None;
12674                    for (k, (value, format_id)) in entries.into_iter().enumerate() {
12675                        let row = row0.saturating_add(k as u32) as usize;
12676                        let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12677                            Some(sheet) => {
12678                                Self::locate_row_in_sheet_for_computed_write_plan(sheet, row)
12679                            }
12680                            None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12681                                row,
12682                                32 * 1024,
12683                            ),
12684                        };
12685                        let key = ComputedWriteChunkKey {
12686                            sheet_id,
12687                            col0,
12688                            chunk_idx,
12689                            chunk_start_row0,
12690                        };
12691                        if segment_key != Some(key)
12692                            && let Some(done) = segment_key.replace(key)
12693                        {
12694                            groups.entry(done).or_default().append(&mut segment);
12695                        }
12696                        segment.push(ComputedWriteChunkEntryPlan {
12697                            row_in_chunk,
12698                            seq,
12699                            value,
12700                            format_id,
12701                        });
12702                    }
12703                    if let Some(done) = segment_key {
12704                        groups.entry(done).or_default().append(&mut segment);
12705                    }
12706                }
12707                ComputedWrite::Rect {
12708                    seq,
12709                    sheet_id,
12710                    sr0,
12711                    sc0,
12712                    values,
12713                } => {
12714                    for (r_off, row) in values.into_iter().enumerate() {
12715                        for (c_off, value) in row.into_iter().enumerate() {
12716                            input_cells = input_cells.saturating_add(1);
12717                            self.push_computed_write_plan_entry(
12718                                &mut groups,
12719                                seq,
12720                                sheet_id,
12721                                sr0.saturating_add(r_off as u32),
12722                                sc0.saturating_add(c_off as u32),
12723                                value,
12724                                None,
12725                            );
12726                        }
12727                    }
12728                }
12729            }
12730        }
12731
12732        let mut plan = ComputedWriteCoalescingPlan {
12733            chunks: Vec::with_capacity(groups.len()),
12734            input_cells,
12735            coalesced_cells: 0,
12736            overwritten_cells: 0,
12737        };
12738        for (key, entries) in groups {
12739            let computed_lane_has_formats =
12740                self.computed_overlay_chunk_has_formats(key.sheet_id, key.col0, key.chunk_idx);
12741            let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(
12742                key,
12743                entries,
12744                formats_present,
12745                computed_lane_has_formats,
12746            );
12747            #[cfg(test)]
12748            if matches!(
12749                &chunk_plan.format_effect,
12750                ComputedWriteChunkFormatEffect::SetExplicit(_)
12751            ) {
12752                self.computed_format_vector_allocations_for_test
12753                    .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12754            }
12755            plan.coalesced_cells = plan
12756                .coalesced_cells
12757                .saturating_add(chunk_plan.entries.len());
12758            plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
12759            plan.chunks.push(chunk_plan);
12760        }
12761        debug_assert_eq!(
12762            plan.input_cells,
12763            plan.coalesced_cells.saturating_add(plan.overwritten_cells)
12764        );
12765        plan
12766    }
12767
12768    fn push_computed_write_plan_entry(
12769        &self,
12770        groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
12771        seq: u64,
12772        sheet_id: SheetId,
12773        row0: u32,
12774        col0: u32,
12775        value: OverlayValue,
12776        format_id: Option<crate::format::FormatId>,
12777    ) {
12778        let (chunk_idx, chunk_start_row0, row_in_chunk) =
12779            self.locate_computed_write_chunk(sheet_id, row0);
12780        let key = ComputedWriteChunkKey {
12781            sheet_id,
12782            col0,
12783            chunk_idx,
12784            chunk_start_row0,
12785        };
12786        groups
12787            .entry(key)
12788            .or_default()
12789            .push(ComputedWriteChunkEntryPlan {
12790                row_in_chunk,
12791                seq,
12792                value,
12793                format_id,
12794            });
12795    }
12796
12797    fn computed_overlay_chunk_has_formats(
12798        &self,
12799        sheet_id: SheetId,
12800        col0: u32,
12801        chunk_idx: usize,
12802    ) -> bool {
12803        let sheet_name = self.graph.sheet_name(sheet_id);
12804        self.arrow_sheets
12805            .sheet(sheet_name)
12806            .and_then(|sheet| sheet.columns.get(col0 as usize))
12807            .and_then(|column| column.chunk(chunk_idx))
12808            .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12809    }
12810
12811    fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
12812        let sheet_name = self.graph.sheet_name(sheet_id);
12813        if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
12814            return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
12815        }
12816        Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
12817    }
12818
12819    fn locate_row_in_sheet_for_computed_write_plan(
12820        sheet: &crate::arrow_store::ArrowSheet,
12821        row0: usize,
12822    ) -> (usize, u32, usize) {
12823        if row0 < sheet.nrows as usize
12824            && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
12825        {
12826            let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
12827            return (chunk_idx, chunk_start as u32, row_in_chunk);
12828        }
12829
12830        let chunk_rows = sheet.chunk_rows.max(1);
12831        if sheet.chunk_starts.is_empty() {
12832            return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
12833        }
12834
12835        let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
12836        let mut chunk_start = sheet.chunk_starts[chunk_idx];
12837        while chunk_start.saturating_add(chunk_rows) <= row0 {
12838            chunk_idx = chunk_idx.saturating_add(1);
12839            chunk_start = chunk_start.saturating_add(chunk_rows);
12840        }
12841        (
12842            chunk_idx,
12843            chunk_start as u32,
12844            row0.saturating_sub(chunk_start),
12845        )
12846    }
12847
12848    fn locate_row_in_empty_sheet_for_computed_write_plan(
12849        row0: usize,
12850        chunk_rows: usize,
12851    ) -> (usize, u32, usize) {
12852        let chunk_rows = chunk_rows.max(1);
12853        let chunk_idx = row0 / chunk_rows;
12854        let chunk_start = chunk_idx.saturating_mul(chunk_rows);
12855        (
12856            chunk_idx,
12857            chunk_start as u32,
12858            row0.saturating_sub(chunk_start),
12859        )
12860    }
12861
12862    #[cfg(test)]
12863    pub(crate) fn debug_plan_computed_write_coalescing(
12864        &self,
12865        buffer: &ComputedWriteBuffer,
12866    ) -> ComputedWriteCoalescingPlan {
12867        self.plan_computed_write_coalescing(buffer)
12868    }
12869
12870    pub(crate) fn flush_computed_write_buffer(
12871        &mut self,
12872        buffer: &mut ComputedWriteBuffer,
12873    ) -> Result<(), ExcelError> {
12874        if buffer.is_empty() {
12875            return Ok(());
12876        }
12877
12878        // Keep ownership of all pending writes until the final request
12879        // checkpoint and bounded commit-window preflight succeed so failures
12880        // remain retry safe. The synchronization and flush that follow are
12881        // infallible mutations with no cancellation point.
12882        self.resource_checkpoint(0)?;
12883        let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
12884        let (writes, formats_present) = buffer.take_writes();
12885        let plan = self.plan_owned_computed_write_coalescing(writes, formats_present);
12886        self.flush_computed_write_plan(plan);
12887        self.observe_evaluation_commit_window(commit_started);
12888
12889        Ok(())
12890    }
12891
12892    fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
12893        for chunk in plan.chunks {
12894            self.flush_computed_write_chunk_plan(chunk);
12895        }
12896    }
12897
12898    fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
12899        match &chunk.shape {
12900            ComputedWriteChunkPlanShape::Point => {
12901                self.flush_computed_write_chunk_plan_as_points(chunk);
12902            }
12903            ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
12904                self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
12905            }
12906            ComputedWriteChunkPlanShape::DenseRange { .. } => {
12907                self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12908            }
12909            ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
12910                if Self::should_emit_computed_run_fragment(*len, *runs) {
12911                    self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
12912                } else {
12913                    self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12914                }
12915            }
12916        }
12917    }
12918
12919    #[inline]
12920    fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
12921        runs <= len / 2
12922    }
12923
12924    fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
12925        let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
12926        for entry in chunk.entries {
12927            let row0 = chunk
12928                .chunk_start_row0
12929                .saturating_add(entry.row_in_chunk as u32);
12930            self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
12931        }
12932        self.apply_computed_overlay_format_effect(
12933            chunk.sheet_id,
12934            chunk.col0,
12935            chunk.chunk_idx,
12936            chunk.format_effect,
12937        );
12938    }
12939
12940    fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
12941        &mut self,
12942        chunk: ComputedWriteChunkPlan,
12943    ) {
12944        let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
12945        let sheet_id = chunk.sheet_id;
12946        let col0 = chunk.col0;
12947        let chunk_idx = chunk.chunk_idx;
12948        let chunk_start_row0 = chunk.chunk_start_row0;
12949        let format_effect = chunk.format_effect;
12950        let items: Vec<(usize, OverlayValue)> = chunk
12951            .entries
12952            .into_iter()
12953            .map(|entry| (entry.row_in_chunk, entry.value))
12954            .collect();
12955        match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
12956            items,
12957            point_estimate,
12958        ) {
12959            Some(Ok(fragment)) => {
12960                self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
12961            }
12962            Some(Err(cells)) => {
12963                self.flush_computed_overlay_cells_as_points(
12964                    sheet_id,
12965                    col0,
12966                    chunk_start_row0,
12967                    cells,
12968                );
12969            }
12970            None => {}
12971        }
12972        self.apply_computed_overlay_format_effect(sheet_id, col0, chunk_idx, format_effect);
12973    }
12974
12975    #[inline]
12976    fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
12977        chunk
12978            .entries
12979            .iter()
12980            .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
12981            .fold(0usize, usize::saturating_add)
12982    }
12983
12984    fn flush_computed_overlay_cells_as_points(
12985        &mut self,
12986        sheet_id: SheetId,
12987        col0: u32,
12988        chunk_start_row0: u32,
12989        cells: Vec<(usize, OverlayValue)>,
12990    ) {
12991        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
12992        for (row_in_chunk, value) in cells {
12993            let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
12994            self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
12995        }
12996    }
12997
12998    fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
12999        if chunk.entries.is_empty() {
13000            return;
13001        }
13002        let start = chunk.entries[0].row_in_chunk;
13003        let values: Vec<OverlayValue> =
13004            chunk.entries.into_iter().map(|entry| entry.value).collect();
13005        if let Some(fragment) = OverlayFragment::dense_range(start, values) {
13006            self.apply_computed_overlay_fragment(
13007                chunk.sheet_id,
13008                chunk.col0,
13009                chunk.chunk_idx,
13010                fragment,
13011            );
13012        }
13013        self.apply_computed_overlay_format_effect(
13014            chunk.sheet_id,
13015            chunk.col0,
13016            chunk.chunk_idx,
13017            chunk.format_effect,
13018        );
13019    }
13020
13021    fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
13022        if chunk.entries.is_empty() {
13023            return;
13024        }
13025        let start = chunk.entries[0].row_in_chunk;
13026        let values: Vec<OverlayValue> =
13027            chunk.entries.into_iter().map(|entry| entry.value).collect();
13028        if let Some(fragment) = OverlayFragment::run_range(start, values) {
13029            self.apply_computed_overlay_fragment(
13030                chunk.sheet_id,
13031                chunk.col0,
13032                chunk.chunk_idx,
13033                fragment,
13034            );
13035        }
13036        self.apply_computed_overlay_format_effect(
13037            chunk.sheet_id,
13038            chunk.col0,
13039            chunk.chunk_idx,
13040            chunk.format_effect,
13041        );
13042    }
13043
13044    fn apply_computed_overlay_format_effect(
13045        &mut self,
13046        sheet_id: SheetId,
13047        col0: u32,
13048        chunk_idx: usize,
13049        effect: ComputedWriteChunkFormatEffect,
13050    ) {
13051        if !(self.config.arrow_storage_enabled
13052            && self.config.delta_overlay_enabled
13053            && self.config.write_formula_overlay_enabled)
13054            || self.computed_overlay_mirroring_disabled
13055        {
13056            return;
13057        }
13058
13059        let sheet_name = self.graph.sheet_name(sheet_id);
13060        let Some(sheet) = self.arrow_sheets.sheet_mut(sheet_name) else {
13061            return;
13062        };
13063        let Some(chunk) = sheet
13064            .columns
13065            .get_mut(col0 as usize)
13066            .and_then(|column| column.chunk_mut(chunk_idx))
13067        else {
13068            return;
13069        };
13070        match effect {
13071            ComputedWriteChunkFormatEffect::NoFormatWork => {}
13072            ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Range {
13073                start,
13074                end,
13075            }) => {
13076                #[cfg(test)]
13077                {
13078                    self.computed_overlay_stale_clear_range_effects_for_test = self
13079                        .computed_overlay_stale_clear_range_effects_for_test
13080                        .saturating_add(1);
13081                }
13082                chunk.computed_overlay.clear_format_range(start, end);
13083            }
13084            ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Offsets(
13085                offsets,
13086            )) => {
13087                #[cfg(test)]
13088                {
13089                    self.computed_overlay_stale_clear_offset_attempts_for_test = self
13090                        .computed_overlay_stale_clear_offset_attempts_for_test
13091                        .saturating_add(offsets.len() as u64);
13092                }
13093                chunk.computed_overlay.clear_format_offsets(&offsets);
13094            }
13095            ComputedWriteChunkFormatEffect::SetExplicit(formats) => {
13096                #[cfg(test)]
13097                {
13098                    self.computed_overlay_set_explicit_entry_operations_for_test = self
13099                        .computed_overlay_set_explicit_entry_operations_for_test
13100                        .saturating_add(formats.len() as u64);
13101                }
13102                for (row_in_chunk, format_id) in formats {
13103                    chunk.computed_overlay.set_format(row_in_chunk, format_id);
13104                }
13105            }
13106        }
13107    }
13108
13109    fn apply_computed_overlay_fragment(
13110        &mut self,
13111        sheet_id: SheetId,
13112        col0: u32,
13113        chunk_idx: usize,
13114        fragment: OverlayFragment,
13115    ) {
13116        if !(self.config.arrow_storage_enabled
13117            && self.config.delta_overlay_enabled
13118            && self.config.write_formula_overlay_enabled)
13119        {
13120            return;
13121        }
13122        if self.computed_overlay_mirroring_disabled {
13123            return;
13124        }
13125
13126        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13127        self.ensure_arrow_sheet(&sheet_name);
13128
13129        let col0 = col0 as usize;
13130        let asheet = self
13131            .arrow_sheets
13132            .sheet_mut(&sheet_name)
13133            .expect("ArrowSheet must exist");
13134
13135        let cur_cols = asheet.columns.len();
13136        if col0 >= cur_cols {
13137            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
13138        }
13139
13140        let start_row0 = asheet
13141            .chunk_starts
13142            .get(chunk_idx)
13143            .copied()
13144            .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
13145        let required_rows =
13146            start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
13147        if required_rows > asheet.nrows as usize {
13148            if asheet.columns.is_empty() {
13149                asheet.insert_columns(0, 1);
13150            }
13151            asheet.ensure_row_capacity(required_rows);
13152        }
13153
13154        let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
13155            return;
13156        };
13157        let delta = ch.computed_overlay.apply_fragment(fragment);
13158        self.adjust_computed_overlay_bytes(delta);
13159
13160        if let Some(cap) = self.config.max_overlay_memory_bytes
13161            && self.computed_overlay_bytes_estimate > cap
13162        {
13163            self.disable_computed_overlay_mirroring_due_to_budget(cap);
13164        }
13165    }
13166
13167    #[inline]
13168    fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
13169        if delta >= 0 {
13170            self.computed_overlay_bytes_estimate = self
13171                .computed_overlay_bytes_estimate
13172                .saturating_add(delta as usize);
13173        } else {
13174            self.computed_overlay_bytes_estimate = self
13175                .computed_overlay_bytes_estimate
13176                .saturating_sub((-delta) as usize);
13177        }
13178    }
13179
13180    fn clear_all_computed_overlays(&mut self) {
13181        let mut freed_total = 0usize;
13182        for sh in self.arrow_sheets.sheets.iter_mut() {
13183            for col in sh.columns.iter_mut() {
13184                for ch in col.chunks.iter_mut() {
13185                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
13186                }
13187                for ch in col.sparse_chunks.values_mut() {
13188                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
13189                }
13190            }
13191        }
13192        self.computed_overlay_bytes_estimate = self
13193            .computed_overlay_bytes_estimate
13194            .saturating_sub(freed_total);
13195    }
13196
13197    fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
13198        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
13199        // Handle budget pressure by compacting computed overlays into base lanes.
13200        self.compact_all_computed_overlays();
13201    }
13202
13203    /// Fold all computed overlay entries across all sheets into their base arrays.
13204    /// This preserves data while freeing overlay memory, allowing mirroring to continue.
13205    fn compact_all_computed_overlays(&mut self) {
13206        let mut freed_total = 0usize;
13207        for sheet in self.arrow_sheets.sheets.iter_mut() {
13208            for col_idx in 0..sheet.columns.len() {
13209                // Dense chunks
13210                let num_dense = sheet.columns[col_idx].chunks.len();
13211                for ch_idx in 0..num_dense {
13212                    freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
13213                }
13214                // Sparse chunks
13215                let sparse_keys: Vec<usize> = sheet.columns[col_idx]
13216                    .sparse_chunks
13217                    .keys()
13218                    .copied()
13219                    .collect();
13220                for ch_idx in sparse_keys {
13221                    freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
13222                }
13223            }
13224        }
13225        self.computed_overlay_bytes_estimate = self
13226            .computed_overlay_bytes_estimate
13227            .saturating_sub(freed_total);
13228        self.overlay_compactions = self.overlay_compactions.saturating_add(1);
13229    }
13230
13231    fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
13232        let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
13233    }
13234
13235    fn record_vertex_value_to_overlay(
13236        &mut self,
13237        vertex_id: VertexId,
13238        value: &LiteralValue,
13239        computed_writes: Option<&mut ComputedWriteBuffer>,
13240    ) -> Result<(), ExcelError> {
13241        if !(self.config.arrow_storage_enabled
13242            && self.config.delta_overlay_enabled
13243            && self.config.write_formula_overlay_enabled)
13244        {
13245            return Ok(());
13246        }
13247        if self.computed_overlay_mirroring_disabled {
13248            return Ok(());
13249        }
13250        if !matches!(
13251            self.graph.get_vertex_kind(vertex_id),
13252            VertexKind::FormulaScalar | VertexKind::FormulaArray
13253        ) {
13254            return Ok(());
13255        }
13256        let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
13257            return Ok(());
13258        };
13259        let Some(buffer) = computed_writes else {
13260            // Unbuffered: one sheet lookup for the value and its format.
13261            let format_id = self.derived_formats.get(&cell);
13262            self.write_computed_cell_0based(cell, value, format_id);
13263            return Ok(());
13264        };
13265        let date_system = self
13266            .arrow_sheet_by_id(cell.sheet_id)
13267            .map(|sheet| sheet.date_system)
13268            .unwrap_or(self.config.date_system);
13269        let ov = Self::literal_to_overlay_value(value, date_system);
13270        {
13271            let format_id = self.derived_formats.get(&cell);
13272            buffer.push_cell_with_format(
13273                cell.sheet_id,
13274                cell.coord.row(),
13275                cell.coord.col(),
13276                ov,
13277                format_id,
13278            );
13279            if self.should_flush_computed_write_buffer(buffer) {
13280                self.flush_computed_write_buffer(buffer)?;
13281            }
13282        }
13283        Ok(())
13284    }
13285
13286    #[inline]
13287    fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
13288        self.config.max_overlay_memory_bytes.is_some_and(|cap| {
13289            if cap == 0 {
13290                return false;
13291            }
13292            self.computed_overlay_bytes_estimate
13293                .saturating_add(buffer.estimated_bytes())
13294                > cap
13295        })
13296    }
13297
13298    /// Estimated memory usage for computed overlays (formula/spill mirroring).
13299    pub fn overlay_memory_usage(&self) -> usize {
13300        self.computed_overlay_bytes_estimate
13301    }
13302
13303    #[cfg(test)]
13304    pub(crate) fn debug_overlay_compactions(&self) -> u64 {
13305        self.overlay_compactions
13306    }
13307
13308    #[cfg(test)]
13309    pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
13310        let mut total = 0usize;
13311        for sheet in &self.arrow_sheets.sheets {
13312            for column in &sheet.columns {
13313                for chunk in &column.chunks {
13314                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13315                }
13316                for chunk in column.sparse_chunks.values() {
13317                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13318                }
13319            }
13320        }
13321        self.computed_overlay_bytes_estimate = total;
13322        total
13323    }
13324
13325    fn resolve_sheet_locator_for_write(
13326        &mut self,
13327        loc: formualizer_common::SheetLocator<'_>,
13328        current_sheet: &str,
13329    ) -> Result<SheetId, ExcelError> {
13330        Ok(match loc {
13331            formualizer_common::SheetLocator::Id(id) => id,
13332            formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
13333            formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
13334        })
13335    }
13336
13337    fn resolve_sheet_locator_for_read(
13338        &self,
13339        loc: formualizer_common::SheetLocator<'_>,
13340        current_sheet: &str,
13341    ) -> Result<SheetId, ExcelError> {
13342        match loc {
13343            formualizer_common::SheetLocator::Id(id) => Ok(id),
13344            formualizer_common::SheetLocator::Name(name) => self
13345                .graph
13346                .sheet_id(name.as_ref())
13347                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13348            formualizer_common::SheetLocator::Current => self
13349                .graph
13350                .sheet_id(current_sheet)
13351                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13352        }
13353    }
13354
13355    /// Set a cell value
13356    pub fn set_cell_value(
13357        &mut self,
13358        sheet: &str,
13359        row: u32,
13360        col: u32,
13361        value: LiteralValue,
13362    ) -> Result<(), ExcelError> {
13363        self.observe_function_semantic_epoch()?;
13364        let sheet_existed = self.graph.sheet_id(sheet).is_some();
13365        let sheet_id = self.graph.sheet_id_mut(sheet);
13366        let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13367        let replaced = self
13368            .graph
13369            .get_vertex_id_for_address(&cell_ref)
13370            .filter(|&vertex| {
13371                matches!(
13372                    self.graph.get_vertex_kind(vertex),
13373                    VertexKind::FormulaScalar | VertexKind::FormulaArray
13374                )
13375            });
13376        let replaced_formula = replaced.is_some();
13377        // A value replacing a spilled anchor takes its spill with it (no
13378        // evaluation will revisit the anchor); anchors that spill blocked
13379        // wake through the cleared region.
13380        if let Some(anchor) = replaced
13381            && self.graph.spill_registry_has_anchor(anchor)
13382        {
13383            self.clear_spill_projection_and_mirror(anchor, None);
13384        }
13385        self.graph.set_cell_value(sheet, row, col, value.clone())?;
13386        self.clear_cell_format_state(sheet, cell_ref);
13387        self.record_changed_cell(sheet, row, col);
13388        if !sheet_existed || replaced_formula {
13389            self.mark_topology_edited();
13390        }
13391        // Mirror into Arrow overlay when enabled
13392        self.mirror_value_to_overlay(sheet, row, col, &value);
13393        // Advance snapshot to reflect external mutation.
13394        self.mark_data_edited();
13395        Ok(())
13396    }
13397
13398    /// Record a single-cell change: invalidates pending spills blocked on it.
13399    fn record_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
13400        let sheet_id = self.graph.sheet_id_mut(sheet);
13401        self.record_structural_change(StructuralScope::Cell {
13402            sheet: sheet_id,
13403            row: row.saturating_sub(1),
13404            col: col.saturating_sub(1),
13405        });
13406    }
13407
13408    fn record_change_for_event(&mut self, event: &ChangeEvent) {
13409        match event {
13410            ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
13411                self.record_structural_change(StructuralScope::Cell {
13412                    sheet: addr.sheet_id,
13413                    row: addr.coord.row(),
13414                    col: addr.coord.col(),
13415                });
13416            }
13417            ChangeEvent::SpillCommitted { new, .. } => {
13418                if let Some(scope) = Self::structural_scope_from_cells(&new.target_cells) {
13419                    self.record_structural_change(scope);
13420                }
13421            }
13422            ChangeEvent::SpillCleared { old, .. } => {
13423                if let Some(scope) = Self::structural_scope_from_cells(&old.target_cells) {
13424                    self.record_structural_change(scope);
13425                }
13426            }
13427            ChangeEvent::DefineName { .. }
13428            | ChangeEvent::UpdateName { .. }
13429            | ChangeEvent::DeleteName { .. }
13430            | ChangeEvent::NamedRangeAdjusted { .. } => {
13431                // Direct name events are preflighted by the logged-name APIs.
13432                // Structural entry points preflight spans before emitting a
13433                // NamedRangeAdjusted event. Epoch changes only rebuild caches.
13434                self.record_structural_change(StructuralScope::AllSheets);
13435            }
13436            ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
13437                // Structural entry points publish their axis delta once after
13438                // the graph and Arrow commits.
13439            }
13440            ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
13441                self.record_structural_change(StructuralScope::Region(Region::whole_row(
13442                    *sheet_id, *row0,
13443                )));
13444            }
13445            ChangeEvent::AddVertex { .. }
13446            | ChangeEvent::RemoveVertex { .. }
13447            | ChangeEvent::EdgeAdded { .. }
13448            | ChangeEvent::EdgeRemoved { .. }
13449            | ChangeEvent::CompoundStart { .. }
13450            | ChangeEvent::CompoundEnd { .. }
13451            | ChangeEvent::StagedFormulaCellChanged { .. } => {}
13452        }
13453    }
13454
13455    fn record_structural_change(&mut self, scope: StructuralScope) {
13456        if let StructuralScope::Cell { sheet, row, col } = scope {
13457            self.wake_spill_owner_of_entered_formula(sheet, row, col);
13458        }
13459        self.wake_spill_blocker_waiters(scope);
13460        self.invalidate_pending_spills(scope);
13461    }
13462
13463    /// Wake anchors waiting on a blocking cell inside `scope`, with their
13464    /// dependents: the anchor's value changes when it spills. A woken anchor
13465    /// registers again if it is still blocked.
13466    fn wake_spill_blocker_waiters(&mut self, scope: StructuralScope) {
13467        if self.spill_blocker_waiters.is_empty() {
13468            return;
13469        }
13470        let woken = self.spill_blocker_waiters.take_affected(scope);
13471        let woken: Vec<VertexId> = woken
13472            .into_iter()
13473            .filter(|&vertex| {
13474                self.graph.vertex_exists(vertex)
13475                    && matches!(
13476                        self.graph.get_vertex_kind(vertex),
13477                        VertexKind::FormulaScalar | VertexKind::FormulaArray
13478                    )
13479            })
13480            .collect();
13481        if !woken.is_empty() {
13482            self.graph.mark_dirty_many(&woken);
13483        }
13484    }
13485
13486    /// A formula entered into a live spill blocks it: wake the owning anchor
13487    /// so its next recalc re-plans (and reports `#SPILL!`). The spill keeps
13488    /// its registry until then; clearing it leaves the formula cell alone.
13489    fn wake_spill_owner_of_entered_formula(&mut self, sheet: SheetId, row: u32, col: u32) {
13490        let cell = self.graph.make_cell_ref_internal(sheet, row, col);
13491        if let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
13492            && self.graph.is_foreign_formula_cell(&cell, owner)
13493        {
13494            self.graph.mark_dirty_many(&[owner]);
13495        }
13496    }
13497
13498    fn structural_scope_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
13499        let first = cells.first()?;
13500        let sheet_id = first.sheet_id;
13501        if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
13502            return Some(StructuralScope::OpaqueGlobal);
13503        }
13504        let mut row_start = first.coord.row();
13505        let mut row_end = row_start;
13506        let mut col_start = first.coord.col();
13507        let mut col_end = col_start;
13508        for cell in cells.iter().skip(1) {
13509            row_start = row_start.min(cell.coord.row());
13510            row_end = row_end.max(cell.coord.row());
13511            col_start = col_start.min(cell.coord.col());
13512            col_end = col_end.max(cell.coord.col());
13513        }
13514        Some(StructuralScope::Region(Region::rect(
13515            sheet_id, row_start, row_end, col_start, col_end,
13516        )))
13517    }
13518
13519    pub fn set_cell_value_ref(
13520        &mut self,
13521        cell: formualizer_common::SheetCellRef<'_>,
13522        current_sheet: &str,
13523        value: LiteralValue,
13524    ) -> Result<(), ExcelError> {
13525        let owned = cell.into_owned();
13526        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13527        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13528        self.set_cell_value(
13529            &sheet_name,
13530            owned.coord.row() + 1,
13531            owned.coord.col() + 1,
13532            value,
13533        )
13534    }
13535
13536    pub fn set_cell_formula_ref(
13537        &mut self,
13538        cell: formualizer_common::SheetCellRef<'_>,
13539        current_sheet: &str,
13540        ast: ASTNode,
13541    ) -> Result<(), ExcelError> {
13542        let owned = cell.into_owned();
13543        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13544        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13545        self.set_cell_formula(
13546            &sheet_name,
13547            owned.coord.row() + 1,
13548            owned.coord.col() + 1,
13549            ast,
13550        )
13551    }
13552
13553    pub fn get_cell_value_ref(
13554        &self,
13555        cell: formualizer_common::SheetCellRef<'_>,
13556        current_sheet: &str,
13557    ) -> Result<Option<LiteralValue>, ExcelError> {
13558        let owned = cell.into_owned();
13559        let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
13560        let sheet_name = self.graph.sheet_name(sheet_id);
13561        Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
13562    }
13563
13564    pub fn resolve_range_view_sheet_ref<'c>(
13565        &'c self,
13566        r: &formualizer_common::SheetRef<'_>,
13567        current_sheet: &str,
13568    ) -> Result<RangeView<'c>, ExcelError> {
13569        use formualizer_common::SheetLocator;
13570
13571        let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
13572            match loc {
13573                SheetLocator::Current => Ok(None),
13574                SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
13575                SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
13576            }
13577        };
13578
13579        let rt = match r {
13580            formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
13581                sheet: sheet_to_opt_name(cell.sheet.clone())?,
13582                row: cell.coord.row() + 1,
13583                col: cell.coord.col() + 1,
13584                row_abs: cell.coord.row_abs(),
13585                col_abs: cell.coord.col_abs(),
13586            },
13587            formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
13588                sheet: sheet_to_opt_name(range.sheet.clone())?,
13589                start_row: range.start_row.map(|b| b.index + 1),
13590                start_col: range.start_col.map(|b| b.index + 1),
13591                end_row: range.end_row.map(|b| b.index + 1),
13592                end_col: range.end_col.map(|b| b.index + 1),
13593                start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
13594                start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
13595                end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
13596                end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
13597            },
13598        };
13599
13600        crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
13601    }
13602
13603    /// Set a cell formula
13604    pub fn set_cell_formula(
13605        &mut self,
13606        sheet: &str,
13607        row: u32,
13608        col: u32,
13609        ast: ASTNode,
13610    ) -> Result<(), ExcelError> {
13611        self.observe_function_semantic_epoch()?;
13612        let sheet_id = self.graph.sheet_id_mut(sheet);
13613        let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13614        let ingested = {
13615            let mut pipeline = self.ingest_pipeline();
13616            pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
13617        };
13618        self.graph.set_cell_formula_with_plan(
13619            sheet,
13620            row,
13621            col,
13622            ingested.ast_id,
13623            &ingested.dep_plan,
13624            ingested.dep_plan.volatile,
13625            ingested.dep_plan.dynamic,
13626        )?;
13627        self.clear_cell_format_state(sheet, placement);
13628        self.record_changed_cell(sheet, row, col);
13629
13630        // If the cell previously held a user value in the delta overlay, it must not continue
13631        // to mask the formula result under Arrow-canonical reads (overlay precedence is
13632        // delta -> computed -> base). Remove the overlay entry instead of writing `Empty`,
13633        // because an explicit `Empty` overlay would still take precedence over computed values.
13634        self.clear_delta_overlay_cell(sheet, row, col);
13635
13636        // Advance snapshot to reflect external mutation
13637        self.mark_topology_edited();
13638        Ok(())
13639    }
13640
13641    /// Bulk set many formulas on a sheet. Skips per-cell snapshot bumping and minimizes edge rebuilds.
13642    pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
13643    where
13644        I: IntoIterator<Item = (u32, u32, ASTNode)>,
13645    {
13646        let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
13647        let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
13648        let sheet_id = self.graph.sheet_id_mut(sheet);
13649        let ingested = {
13650            let mut pipeline = self.ingest_pipeline();
13651            let inputs = collected.into_iter().map(|(row, col, ast)| {
13652                let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13653                (FormulaAstInput::Tree(ast), placement, None)
13654            });
13655            pipeline.ingest_batch(inputs)?
13656        };
13657        let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
13658            .into_iter()
13659            .map(|formula| {
13660                (
13661                    formula.placement.coord.row() + 1,
13662                    formula.placement.coord.col() + 1,
13663                    formula.ast_id,
13664                    formula.dep_plan,
13665                )
13666            })
13667            .collect();
13668        let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
13669        for (row, col) in edited_cells {
13670            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13671            self.clear_cell_format_state(sheet, cell);
13672            self.record_changed_cell(sheet, row, col);
13673        }
13674        // Single topology bump after batch
13675        if n > 0 {
13676            self.mark_topology_edited();
13677        }
13678        Ok(n)
13679    }
13680
13681    /// Declare the formula cell at `sheet!(row, col)` (1-based) a dynamic
13682    /// array anchor whose spill identity comes from the source document, for
13683    /// example an XLSX `t="array"` formula bound to XLDAPR cell metadata.
13684    ///
13685    /// This changes only spill references to the anchor (`A1#`,
13686    /// `_xlfn.ANCHORARRAY(A1)`). A declared anchor with a committed
13687    /// multi-cell spill resolves to that spill, exactly as an undeclared
13688    /// one. Without a committed spill, a declared anchor whose current
13689    /// result is a scalar (not an error, so not a blocked `#SPILL!`
13690    /// anchor, and not empty) resolves to its own cell as a 1x1 reference,
13691    /// matching a spilled 1x1 dynamic array. Every other case keeps the
13692    /// `#REF!` of an undeclared anchor. Undeclared cells are unaffected, and
13693    /// fresh 1x1 results are not given spill identity by function name.
13694    ///
13695    /// The declaration belongs to the formula vertex. It is dropped when the
13696    /// cell's formula is replaced, the cell is cleared or overwritten with a
13697    /// value, the vertex is removed (including with its sheet), or a
13698    /// structural row/column edit moves the vertex. A structural edit that
13699    /// only adjusts the formula's references, without moving the anchor,
13700    /// keeps it. Declaring again re-establishes it.
13701    ///
13702    /// The anchor and its dependents are marked dirty, so the next
13703    /// evaluation observes the declaration.
13704    ///
13705    /// Returns `#REF!` if the sheet does not exist or the cell holds no
13706    /// formula vertex. A formula still staged by deferred graph building has
13707    /// no vertex: build the graph (for example [`Self::build_graph_all`])
13708    /// before declaring.
13709    pub fn declare_dynamic_array_anchor(
13710        &mut self,
13711        sheet: &str,
13712        row: u32,
13713        col: u32,
13714    ) -> Result<(), ExcelError> {
13715        let not_a_formula = || {
13716            ExcelError::new(ExcelErrorKind::Ref)
13717                .with_message("Declared dynamic array anchor must be a formula cell")
13718        };
13719        if row == 0 || col == 0 {
13720            return Err(not_a_formula());
13721        }
13722        if self.get_staged_formula_text(sheet, row, col).is_some() {
13723            return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
13724                "Declared dynamic array anchor is still staged; build the graph first",
13725            ));
13726        }
13727        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(not_a_formula)?;
13728        let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13729        let vertex = self
13730            .graph
13731            .cell_vertex_mut(&cell)
13732            .filter(|v| self.graph.vertex_has_formula(*v))
13733            .ok_or_else(not_a_formula)?;
13734        if self.graph.declare_dynamic_anchor(vertex, cell) {
13735            self.graph.mark_dirty_many(&[vertex]);
13736            self.mark_topology_edited();
13737        }
13738        Ok(())
13739    }
13740
13741    /// Declare a legacy array formula with a fixed output rectangle after
13742    /// ingestion and before evaluation. Coordinates are one-based. Replacement,
13743    /// removal or movement of the formula clears the declaration.
13744    /// Single-cell declarations do not register a spill anchor. Requires
13745    /// `EvalConfig::family_execution = false` to bypass family memoization.
13746    pub fn declare_fixed_array_formula(
13747        &mut self,
13748        sheet: &str,
13749        row: u32,
13750        col: u32,
13751        rows: u32,
13752        cols: u32,
13753    ) -> Result<(), ExcelError> {
13754        if self.config.family_execution {
13755            return Err(ExcelError::new(ExcelErrorKind::Ref)
13756                .with_message("Fixed arrays require family_execution = false"));
13757        }
13758        if row == 0
13759            || col == 0
13760            || rows == 0
13761            || cols == 0
13762            || row.checked_add(rows - 1).is_none_or(|r| r > 1_048_576)
13763            || col.checked_add(cols - 1).is_none_or(|c| c > 16_384)
13764            || u64::from(rows) * u64::from(cols) > u64::from(self.config.spill.max_spill_cells)
13765        {
13766            return Err(ExcelError::new(ExcelErrorKind::Ref)
13767                .with_message("Invalid or over-cap fixed array extent"));
13768        }
13769        if self.get_staged_formula_text(sheet, row, col).is_some() {
13770            return Err(ExcelError::new(ExcelErrorKind::Ref)
13771                .with_message("Fixed array formula is still staged; build the graph first"));
13772        }
13773        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
13774            ExcelError::new(ExcelErrorKind::Ref).with_message("Unknown fixed array sheet")
13775        })?;
13776        let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13777        let vertex = self
13778            .graph
13779            .cell_vertex_mut(&cell)
13780            .filter(|v| self.graph.vertex_has_formula(*v))
13781            .ok_or_else(|| {
13782                ExcelError::new(ExcelErrorKind::Ref)
13783                    .with_message("Fixed array anchor must be a formula cell")
13784            })?;
13785        if (rows == 1 && cols == 1 && self.graph.fixed_single_arrays.contains(&vertex))
13786            || self.graph.fixed_array_shapes.get(&vertex) == Some(&(rows, cols))
13787        {
13788            return Ok(());
13789        }
13790        if self.graph.fixed_single_arrays.contains(&vertex)
13791            || self.graph.spill_registry_has_anchor(vertex)
13792            || self.graph.is_current_declared_dynamic_anchor(vertex)
13793        {
13794            return Err(ExcelError::new(ExcelErrorKind::Ref)
13795                .with_message("Array declaration already exists; replace the formula first"));
13796        }
13797        if rows == 1 && cols == 1 {
13798            self.graph.fixed_single_arrays.insert(vertex);
13799        } else {
13800            let mut targets = Vec::with_capacity((rows * cols) as usize);
13801            for r in 0..rows {
13802                for c in 0..cols {
13803                    targets.push(self.graph.make_cell_ref(sheet, row + r, col + c));
13804                }
13805            }
13806            self.graph.plan_spill_region(vertex, &targets)?;
13807            // Reserve ownership before evaluation, even for Empty members.
13808            self.graph
13809                .commit_spill_region_atomic_with_fault(vertex, targets, vec![], None)?;
13810            self.graph.fixed_array_shapes.insert(vertex, (rows, cols));
13811        }
13812        self.graph.mark_dirty_many(&[vertex]);
13813        self.mark_topology_edited();
13814        Ok(())
13815    }
13816
13817    #[inline]
13818    fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
13819        match v {
13820            LiteralValue::Empty => None,
13821            LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
13822            other => Some(other),
13823        }
13824    }
13825
13826    fn materialize_temporal_egress(
13827        value: LiteralValue,
13828        class: Option<&formualizer_common::numfmt::FormatClass>,
13829        policy: crate::engine::TemporalEgress,
13830        date_system: crate::engine::DateSystem,
13831    ) -> LiteralValue {
13832        use formualizer_common::numfmt::FormatClass;
13833        if policy == crate::engine::TemporalEgress::Serial {
13834            return value;
13835        }
13836        let LiteralValue::Number(serial) = value else {
13837            return value;
13838        };
13839        match class {
13840            Some(FormatClass::Date) if serial.fract() != 0.0 => {
13841                formualizer_common::try_serial_to_datetime_for(date_system, serial)
13842                    .map(LiteralValue::DateTime)
13843                    .unwrap_or(LiteralValue::Number(serial))
13844            }
13845            Some(FormatClass::Date) => {
13846                formualizer_common::try_serial_to_date_for(date_system, serial)
13847                    .map(LiteralValue::Date)
13848                    .unwrap_or(LiteralValue::Number(serial))
13849            }
13850            Some(FormatClass::DateTime) => {
13851                formualizer_common::try_serial_to_datetime_for(date_system, serial)
13852                    .map(LiteralValue::DateTime)
13853                    .unwrap_or(LiteralValue::Number(serial))
13854            }
13855            Some(FormatClass::Time) if (0.0..1.0).contains(&serial) => {
13856                let nanos = (serial * 86_400_000_000_000.0).round() as u64;
13857                let nanos = nanos.min(86_400_000_000_000 - 1);
13858                chrono::NaiveTime::from_num_seconds_from_midnight_opt(
13859                    (nanos / 1_000_000_000) as u32,
13860                    (nanos % 1_000_000_000) as u32,
13861                )
13862                .map(LiteralValue::Time)
13863                .unwrap_or(LiteralValue::Number(serial))
13864            }
13865            Some(FormatClass::Time | FormatClass::Duration) => {
13866                let nanos = (serial * 86_400.0 * 1_000_000_000.0).round();
13867                if nanos.is_finite() && nanos >= i64::MIN as f64 && nanos <= i64::MAX as f64 {
13868                    LiteralValue::Duration(chrono::Duration::nanoseconds(nanos as i64))
13869                } else {
13870                    LiteralValue::Number(serial)
13871                }
13872            }
13873            _ => LiteralValue::Number(serial),
13874        }
13875    }
13876
13877    pub(crate) fn effective_format_id(
13878        &self,
13879        sheet: &str,
13880        row: u32,
13881        col: u32,
13882    ) -> Option<crate::format::FormatId> {
13883        let arrow = self.arrow_sheets.sheet(sheet).and_then(|arrow| {
13884            arrow.format_id(
13885                row.saturating_sub(1) as usize,
13886                col.saturating_sub(1) as usize,
13887            )
13888        });
13889        arrow.or_else(|| {
13890            let sheet_id = self.graph.sheet_id(sheet)?;
13891            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13892            self.derived_formats.get(&cell)
13893        })
13894    }
13895
13896    /// Get a cell value through the single temporal egress boundary.
13897    pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13898        let raw = self.read_cell_value(sheet, row, col)?;
13899        let format = self.effective_format_id(sheet, row, col);
13900        let class = format.and_then(|id| self.format_registry.class(id));
13901        Self::normalize_public_cell_read(Self::materialize_temporal_egress(
13902            raw,
13903            class,
13904            self.config.temporal_egress,
13905            self.config.date_system,
13906        ))
13907    }
13908
13909    /// Read a rectangular range through the temporal egress boundary.
13910    pub fn get_range_values(
13911        &self,
13912        sheet: &str,
13913        sr: u32,
13914        sc: u32,
13915        er: u32,
13916        ec: u32,
13917    ) -> Vec<Vec<LiteralValue>> {
13918        let height = er.saturating_sub(sr).saturating_add(1) as usize;
13919        let width = ec.saturating_sub(sc).saturating_add(1) as usize;
13920        let Some(asheet) = self.sheet_store().sheet(sheet) else {
13921            return vec![vec![LiteralValue::Empty; width]; height];
13922        };
13923        let view = asheet.range_view(
13924            sr.saturating_sub(1) as usize,
13925            sc.saturating_sub(1) as usize,
13926            er.saturating_sub(1) as usize,
13927            ec.saturating_sub(1) as usize,
13928        );
13929        let sheet_id = self.graph.sheet_id(sheet);
13930        let derived_formats = &self.derived_formats;
13931        let has_derived_formats =
13932            sheet_id.is_some_and(|sheet_id| derived_formats.any(|cell| cell.sheet_id == sheet_id));
13933        let mut out = Vec::with_capacity(height);
13934        if !asheet.has_formats() && !has_derived_formats {
13935            for rr in 0..height {
13936                let mut row = Vec::with_capacity(width);
13937                for cc in 0..width {
13938                    row.push(view.get_cell(rr, cc));
13939                }
13940                out.push(row);
13941            }
13942            return out;
13943        }
13944        let format_registry = &self.format_registry;
13945        for rr in 0..height {
13946            let mut row = Vec::with_capacity(width);
13947            for cc in 0..width {
13948                let raw = view.get_cell(rr, cc);
13949                let row0 = sr.saturating_sub(1).saturating_add(rr as u32);
13950                let col0 = sc.saturating_sub(1).saturating_add(cc as u32);
13951                let format = asheet.format_id(row0 as usize, col0 as usize).or_else(|| {
13952                    let cell = CellRef::new(sheet_id?, Coord::new(row0, col0, true, true));
13953                    derived_formats.get(&cell)
13954                });
13955                let class = format.and_then(|id| format_registry.class(id));
13956                row.push(Self::materialize_temporal_egress(
13957                    raw,
13958                    class,
13959                    self.config.temporal_egress,
13960                    self.config.date_system,
13961                ));
13962            }
13963            out.push(row);
13964        }
13965        out
13966    }
13967
13968    /// Unified internal read API for a single cell value (Arrow-truth).
13969    /// The 1x1 spill fallback of a declared dynamic-array anchor: its
13970    /// committed result is a non-error, non-empty scalar. Scheduling orders
13971    /// the anchor before its spill readers (the reference keeps a static
13972    /// edge to it), exactly as for committed multi-cell spills.
13973    fn declared_anchor_holds_scalar(&self, anchor: CellRef) -> bool {
13974        let sheet = self.graph.sheet_name(anchor.sheet_id);
13975        if sheet.is_empty() {
13976            return false;
13977        }
13978        matches!(
13979            self.read_cell_value(sheet, anchor.coord.row() + 1, anchor.coord.col() + 1),
13980            Some(value) if !matches!(
13981                value,
13982                LiteralValue::Error(_)
13983                    | LiteralValue::Array(_)
13984                    | LiteralValue::Pending
13985                    | LiteralValue::Empty
13986            )
13987        )
13988    }
13989
13990    pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13991        let asheet = self.sheet_store().sheet(sheet)?;
13992        let r0 = row.saturating_sub(1) as usize;
13993        let c0 = col.saturating_sub(1) as usize;
13994        let v = asheet.get_cell_value(r0, c0);
13995        if matches!(v, LiteralValue::Empty) {
13996            None
13997        } else {
13998            Some(v)
13999        }
14000    }
14001
14002    /// Unified internal read API for a range of cell values (Arrow-truth).
14003    pub(crate) fn read_range_values(
14004        &self,
14005        sheet: &str,
14006        sr: u32,
14007        sc: u32,
14008        er: u32,
14009        ec: u32,
14010    ) -> RangeView<'_> {
14011        let Some(asheet) = self.sheet_store().sheet(sheet) else {
14012            return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
14013        };
14014        if er < sr || ec < sc {
14015            return asheet.range_view(1, 1, 0, 0);
14016        }
14017        let sr0 = sr.saturating_sub(1) as usize;
14018        let sc0 = sc.saturating_sub(1) as usize;
14019        let er0 = er.saturating_sub(1) as usize;
14020        let ec0 = ec.saturating_sub(1) as usize;
14021        asheet.range_view(sr0, sc0, er0, ec0)
14022    }
14023
14024    /// Get formula AST (if any) and current stored value for a cell
14025    pub fn get_cell(
14026        &self,
14027        sheet: &str,
14028        row: u32,
14029        col: u32,
14030    ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
14031        let v = self.get_cell_value(sheet, row, col);
14032        let sheet_id = self.graph.sheet_id(sheet)?;
14033        let coord = Coord::from_excel(row, col, true, true);
14034        let cell = CellRef::new(sheet_id, coord);
14035        if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
14036            let ast = self.graph.get_formula(vid);
14037            Some((ast, v))
14038        } else if v.is_some() || self.graph.had_legacy_cell_vertex(&cell) {
14039            // A referenced or emptied value cell has no vertex (decision
14040            // 27), but it is a cell the graph knows, as it was when it had
14041            // one (the interactive formula edit routes on this).
14042            Some((None, v))
14043        } else {
14044            None
14045        }
14046    }
14047
14048    /// Begin batch operations - defer CSR rebuilds for better performance
14049    pub fn begin_batch(&mut self) {
14050        self.graph.begin_batch();
14051    }
14052
14053    /// End batch operations and trigger CSR rebuild
14054    pub fn end_batch(&mut self) {
14055        self.graph.end_batch();
14056    }
14057
14058    /// Begin a deferred-dirty scope for a multi-edit batch: while active,
14059    /// every edit's dirty propagation queues its sources instead of running
14060    /// a full BFS per edit, and the outermost `end_deferred_dirty` flushes
14061    /// the union with ONE multi-source propagation (O(component) instead of
14062    /// O(edits × component)). See `DependencyGraph::begin_deferred_dirty`.
14063    ///
14064    /// Callers MUST run `end_deferred_dirty` on every exit path, including
14065    /// error returns; evaluation entry points `debug_assert` no scope leaked.
14066    pub fn begin_deferred_dirty(&mut self) {
14067        self.graph.begin_deferred_dirty();
14068    }
14069
14070    /// End a deferred-dirty scope, flushing the queued propagation when the
14071    /// outermost scope closes. See `Engine::begin_deferred_dirty`.
14072    pub fn end_deferred_dirty(&mut self) {
14073        let _ = self.graph.end_deferred_dirty();
14074    }
14075
14076    /// Total vertices processed by dirty-propagation BFS loops since graph
14077    /// creation. Perf-shape observability only (cross-crate tests assert
14078    /// batched edits propagate O(component), not O(edits × component)).
14079    pub fn dirty_propagation_visits(&self) -> u64 {
14080        self.graph.dirty_propagation_visits()
14081    }
14082
14083    /// Evaluate a single vertex.
14084    /// This is the core of the sequential evaluation logic for Milestone 3.1.
14085    #[inline]
14086    fn record_cell_if_changed(
14087        delta: &mut DeltaCollector,
14088        cell: &CellRef,
14089        old: &LiteralValue,
14090        new: &LiteralValue,
14091    ) {
14092        if old != new {
14093            delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
14094        }
14095    }
14096
14097    pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
14098        self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
14099            engine.observe_function_semantic_epoch()?;
14100            // A direct request selects exactly one vertex, regardless of its formula kind.
14101            engine.resource_checkpoint(1)?;
14102            if !engine.graph.vertex_exists(vertex_id) {
14103                return engine.evaluate_vertex_impl(vertex_id, None);
14104            }
14105            let is_formula = matches!(
14106                engine.graph.get_vertex_kind(vertex_id),
14107                VertexKind::FormulaScalar | VertexKind::FormulaArray
14108            );
14109            if is_formula {
14110                engine.begin_evaluation_request();
14111                #[cfg(any(test, feature = "legacy_oracle"))]
14112                engine.graph.flush_pending_edge_deltas();
14113                let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
14114                    vertex_id,
14115                )];
14116                engine.evaluate_legacy_target_roots(&roots, None)?;
14117            }
14118            if !is_formula {
14119                engine.refresh_resolution_snapshots();
14120            }
14121            engine.evaluate_vertex_impl(vertex_id, None)
14122        })
14123    }
14124
14125    /// Same rejection publication for owned arrays and pre-admitted range views.
14126    fn publish_oversized_spill(
14127        &mut self,
14128        vertex_id: VertexId,
14129        error: ExcelError,
14130        mut delta: Option<&mut DeltaCollector>,
14131    ) -> LiteralValue {
14132        self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14133        let anchor = self
14134            .graph
14135            .get_cell_ref(vertex_id)
14136            .expect("cell ref for vertex");
14137        let spill_val = LiteralValue::Error(error);
14138        if let Some(d) = delta {
14139            let old = self
14140                .read_cell_value(
14141                    self.graph.sheet_name(anchor.sheet_id),
14142                    anchor.coord.row() + 1,
14143                    anchor.coord.col() + 1,
14144                )
14145                .unwrap_or(LiteralValue::Empty);
14146            if old != spill_val {
14147                d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14148            }
14149        }
14150        self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14151        if self.config.arrow_storage_enabled
14152            && self.config.delta_overlay_enabled
14153            && self.config.write_formula_overlay_enabled
14154        {
14155            let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14156            self.mirror_value_to_computed_overlay(
14157                &sheet_name,
14158                anchor.coord.row() + 1,
14159                anchor.coord.col() + 1,
14160                &spill_val,
14161            );
14162        }
14163        spill_val
14164    }
14165
14166    fn evaluate_vertex_impl(
14167        &mut self,
14168        vertex_id: VertexId,
14169        delta: Option<&mut DeltaCollector>,
14170    ) -> Result<LiteralValue, ExcelError> {
14171        // Preserve the direct evaluator's compatibility behavior for invalid IDs, literal cells,
14172        // names, and other non-formula vertices. Only formula publication needs the C1a final
14173        // deadline checkpoint and effects pipeline.
14174        if !self.graph.vertex_exists(vertex_id) {
14175            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
14176                .with_message(format!("Vertex not found: {vertex_id:?}")));
14177        }
14178        if (self.active_resource_ledger.is_some()
14179            || self.graph.fixed_array_shapes.contains_key(&vertex_id))
14180            && matches!(
14181                self.graph.get_vertex_kind(vertex_id),
14182                VertexKind::FormulaScalar | VertexKind::FormulaArray
14183            )
14184        {
14185            let value = self
14186                .evaluate_vertex_immutable(vertex_id)
14187                .unwrap_or_else(LiteralValue::Error);
14188            let effects = self.plan_vertex_effects(vertex_id, value.clone())?;
14189            // Do not publish the selected result until the outer request's deadline succeeds.
14190            self.resource_checkpoint(0)?;
14191            let mut delta = delta;
14192            for effect in &effects {
14193                self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
14194            }
14195            return Ok(value);
14196        }
14197
14198        let mut delta = delta;
14199
14200        // Get vertex kind and check if it needs evaluation
14201        let kind = self.graph.get_vertex_kind(vertex_id);
14202        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
14203
14204        let view = match kind {
14205            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
14206                if let Some(view) = self.graph.formula_view(vertex_id) {
14207                    view
14208                } else {
14209                    return Ok(LiteralValue::Number(0.0));
14210                }
14211            }
14212            VertexKind::Empty | VertexKind::Cell => {
14213                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
14214                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14215                    let row = cell_ref.coord.row() + 1;
14216                    let col = cell_ref.coord.col() + 1;
14217                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
14218                        return Ok(v);
14219                    }
14220                }
14221                return Ok(LiteralValue::Number(0.0));
14222            }
14223            VertexKind::NamedScalar => {
14224                let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
14225                return Ok(value);
14226            }
14227            VertexKind::NamedArray => {
14228                let value = self.evaluate_named_array(vertex_id, sheet_id)?;
14229                return Ok(value);
14230            }
14231            VertexKind::InfiniteRange
14232            | VertexKind::Range
14233            | VertexKind::External
14234            | VertexKind::Table => {
14235                // Not directly evaluatable here.
14236                return Ok(LiteralValue::Number(0.0));
14237            }
14238        };
14239
14240        // The interpreter uses a reference to the engine as the context.
14241        let sheet_name = self.graph.sheet_name(sheet_id);
14242        let cell_ref = self
14243            .graph
14244            .get_cell_ref(vertex_id)
14245            .expect("cell ref for vertex");
14246        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14247            .with_resolved_functions(&self.resolved_functions)
14248            .with_arena_cells(self);
14249
14250        let result = interpreter.evaluate_formula_view(
14251            view,
14252            self.graph.data_store(),
14253            self.graph.sheet_reg(),
14254        );
14255
14256        // If array result, perform spill from the anchor cell
14257        match result {
14258            Ok(cv) => {
14259                let derived_format = cv.format_id();
14260                self.record_derived_format(vertex_id, derived_format);
14261                let oversized_range = if matches!(cv, crate::traits::CalcValue::Range(_))
14262                    && self.graph.fixed_single_arrays.contains(&vertex_id)
14263                {
14264                    None
14265                } else {
14266                    crate::engine::result_finalization::range_spill_error(
14267                        &cv,
14268                        self.config.spill.max_spill_cells,
14269                    )
14270                };
14271                let is_oversized_range = oversized_range.is_some();
14272                let result_literal = if let Some(error) = oversized_range {
14273                    drop(cv);
14274                    LiteralValue::Error(error)
14275                } else {
14276                    self.materialize_formula_result(vertex_id, cv)
14277                };
14278                let output_sheet_name = sheet_name.to_string();
14279                self.write_computed_overlay_format_0based(
14280                    &output_sheet_name,
14281                    cell_ref.coord.row(),
14282                    cell_ref.coord.col(),
14283                    derived_format,
14284                );
14285                if is_oversized_range {
14286                    let LiteralValue::Error(error) = result_literal else {
14287                        unreachable!()
14288                    };
14289                    self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
14290                    return Ok(self.publish_oversized_spill(
14291                        vertex_id,
14292                        error,
14293                        delta.as_deref_mut(),
14294                    ));
14295                }
14296                match result_literal {
14297                    LiteralValue::Array(rows) => {
14298                        // Update kind to FormulaArray for tracking
14299                        self.graph
14300                            .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
14301                        // Build target cells rectangle starting from anchor
14302                        let anchor = self
14303                            .graph
14304                            .get_cell_ref(vertex_id)
14305                            .expect("cell ref for vertex");
14306                        let sheet_id = anchor.sheet_id;
14307                        let h = rows.len() as u32;
14308                        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
14309
14310                        // Hard cap to avoid vertex explosion from huge dynamic arrays.
14311                        let spill_cells = (h as u64).saturating_mul(w as u64);
14312                        if spill_cells > self.config.spill.max_spill_cells as u64 {
14313                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14314                                .with_message("SpillTooLarge")
14315                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14316                                    expected_rows: h,
14317                                    expected_cols: w,
14318                                });
14319                            return Ok(self.publish_oversized_spill(
14320                                vertex_id,
14321                                spill_err,
14322                                delta.as_deref_mut(),
14323                            ));
14324                        }
14325                        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity)
14326                        const PACKED_MAX_ROW: u32 = 1_048_575; // 20-bit max
14327                        const PACKED_MAX_COL: u32 = 16_383; // 14-bit max
14328                        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
14329                        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
14330                        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
14331                            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14332                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14333                                .with_message("Spill exceeds sheet bounds")
14334                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14335                                    expected_rows: h,
14336                                    expected_cols: w,
14337                                });
14338                            let spill_val = LiteralValue::Error(spill_err.clone());
14339                            if let Some(d) = delta.as_deref_mut() {
14340                                let old = self
14341                                    .read_cell_value(
14342                                        self.graph.sheet_name(anchor.sheet_id),
14343                                        anchor.coord.row() + 1,
14344                                        anchor.coord.col() + 1,
14345                                    )
14346                                    .unwrap_or(LiteralValue::Empty);
14347                                if old != spill_val {
14348                                    d.record_cell(
14349                                        anchor.sheet_id,
14350                                        anchor.coord.row(),
14351                                        anchor.coord.col(),
14352                                    );
14353                                }
14354                            }
14355                            self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14356                            if self.config.arrow_storage_enabled
14357                                && self.config.delta_overlay_enabled
14358                                && self.config.write_formula_overlay_enabled
14359                            {
14360                                let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14361                                self.mirror_value_to_computed_overlay(
14362                                    &sheet_name,
14363                                    anchor.coord.row() + 1,
14364                                    anchor.coord.col() + 1,
14365                                    &spill_val,
14366                                );
14367                            }
14368                            return Ok(spill_val);
14369                        }
14370                        let mut targets = Vec::new();
14371                        for r in 0..h {
14372                            for c in 0..w {
14373                                targets.push(self.graph.make_cell_ref_internal(
14374                                    sheet_id,
14375                                    anchor.coord.row() + r,
14376                                    anchor.coord.col() + c,
14377                                ));
14378                            }
14379                        }
14380
14381                        // Plan spill via spill manager shim
14382                        match self.spill_mgr.reserve(
14383                            vertex_id,
14384                            anchor,
14385                            SpillShape { rows: h, cols: w },
14386                            SpillMeta {
14387                                epoch: self.recalc_epoch,
14388                                config: self.config.spill,
14389                            },
14390                        ) {
14391                            Ok(()) => {
14392                                // Commit: write values to grid
14393                                // Default conflict policy is Error + FirstWins; reserve() enforces in-flight locks
14394                                // and plan_spill_region enforces overlap with committed formulas/spills/values.
14395                                if let Err(e) = self.commit_spill_and_mirror(
14396                                    vertex_id,
14397                                    &targets,
14398                                    rows.clone(),
14399                                    delta.as_deref_mut(),
14400                                ) {
14401                                    if e.kind != ExcelErrorKind::Spill {
14402                                        return Err(e);
14403                                    }
14404                                    // If commit fails, mark as error
14405                                    self.clear_spill_projection_and_mirror(
14406                                        vertex_id,
14407                                        delta.as_deref_mut(),
14408                                    );
14409                                    if let Some(d) = delta.as_deref_mut() {
14410                                        let old = self
14411                                            .read_cell_value(
14412                                                self.graph.sheet_name(anchor.sheet_id),
14413                                                anchor.coord.row() + 1,
14414                                                anchor.coord.col() + 1,
14415                                            )
14416                                            .unwrap_or(LiteralValue::Empty);
14417                                        let new = LiteralValue::Error(e.clone());
14418                                        if old != new {
14419                                            d.record_cell(
14420                                                anchor.sheet_id,
14421                                                anchor.coord.row(),
14422                                                anchor.coord.col(),
14423                                            );
14424                                        }
14425                                    }
14426                                    let err_val = LiteralValue::Error(e.clone());
14427                                    self.graph.update_vertex_value_ref(vertex_id, &err_val);
14428                                    if self.config.arrow_storage_enabled
14429                                        && self.config.delta_overlay_enabled
14430                                        && self.config.write_formula_overlay_enabled
14431                                    {
14432                                        let sheet_name =
14433                                            self.graph.sheet_name(anchor.sheet_id).to_string();
14434                                        self.mirror_value_to_computed_overlay(
14435                                            &sheet_name,
14436                                            anchor.coord.row() + 1,
14437                                            anchor.coord.col() + 1,
14438                                            &err_val,
14439                                        );
14440                                    }
14441                                    return Ok(err_val);
14442                                }
14443                                // Anchor shows the top-left value, like Excel
14444                                let top_left = rows
14445                                    .first()
14446                                    .and_then(|r| r.first())
14447                                    .cloned()
14448                                    .unwrap_or(LiteralValue::Empty);
14449                                self.graph.update_vertex_value_ref(vertex_id, &top_left);
14450                                Ok(top_left)
14451                            }
14452                            Err(e) => {
14453                                self.clear_spill_projection_and_mirror(
14454                                    vertex_id,
14455                                    delta.as_deref_mut(),
14456                                );
14457                                let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14458                                    .with_message(e.message().unwrap_or("Spill blocked"))
14459                                    .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14460                                        expected_rows: h,
14461                                        expected_cols: w,
14462                                    });
14463                                let spill_val = LiteralValue::Error(spill_err.clone());
14464                                if let Some(d) = delta.as_deref_mut() {
14465                                    let old = self
14466                                        .read_cell_value(
14467                                            self.graph.sheet_name(anchor.sheet_id),
14468                                            anchor.coord.row() + 1,
14469                                            anchor.coord.col() + 1,
14470                                        )
14471                                        .unwrap_or(LiteralValue::Empty);
14472                                    if old != spill_val {
14473                                        d.record_cell(
14474                                            anchor.sheet_id,
14475                                            anchor.coord.row(),
14476                                            anchor.coord.col(),
14477                                        );
14478                                    }
14479                                }
14480                                self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14481                                if self.config.arrow_storage_enabled
14482                                    && self.config.delta_overlay_enabled
14483                                    && self.config.write_formula_overlay_enabled
14484                                {
14485                                    let sheet_name =
14486                                        self.graph.sheet_name(anchor.sheet_id).to_string();
14487                                    self.mirror_value_to_computed_overlay(
14488                                        &sheet_name,
14489                                        anchor.coord.row() + 1,
14490                                        anchor.coord.col() + 1,
14491                                        &spill_val,
14492                                    );
14493                                }
14494                                Ok(spill_val)
14495                            }
14496                        }
14497                    }
14498                    other => {
14499                        // Scalar result: store value and ensure any previous spill is cleared
14500                        let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14501                        if let Some(d) = delta.as_deref_mut()
14502                            && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14503                        {
14504                            if spill_cells.is_empty() {
14505                                let old = self
14506                                    .read_cell_value(
14507                                        self.graph.sheet_name(anchor.sheet_id),
14508                                        anchor.coord.row() + 1,
14509                                        anchor.coord.col() + 1,
14510                                    )
14511                                    .unwrap_or(LiteralValue::Empty);
14512                                if old != other {
14513                                    d.record_cell(
14514                                        anchor.sheet_id,
14515                                        anchor.coord.row(),
14516                                        anchor.coord.col(),
14517                                    );
14518                                }
14519                            } else {
14520                                for cell in spill_cells.iter() {
14521                                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
14522                                    let old = self
14523                                        .read_cell_value(
14524                                            sheet_name,
14525                                            cell.coord.row() + 1,
14526                                            cell.coord.col() + 1,
14527                                        )
14528                                        .unwrap_or(LiteralValue::Empty);
14529                                    let new = if cell.sheet_id == anchor.sheet_id
14530                                        && cell.coord.row() == anchor.coord.row()
14531                                        && cell.coord.col() == anchor.coord.col()
14532                                    {
14533                                        other.clone()
14534                                    } else {
14535                                        LiteralValue::Empty
14536                                    };
14537                                    Self::record_cell_if_changed(d, cell, &old, &new);
14538                                }
14539                            }
14540                        }
14541                        self.graph.clear_spill_region(vertex_id);
14542                        if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14543                            self.record_structural_change(scope);
14544                        }
14545                        if self.config.arrow_storage_enabled
14546                            && self.config.delta_overlay_enabled
14547                            && self.config.write_formula_overlay_enabled
14548                        {
14549                            let empty = LiteralValue::Empty;
14550                            for cell in spill_cells.iter() {
14551                                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14552                                self.mirror_value_to_computed_overlay(
14553                                    &sheet_name,
14554                                    cell.coord.row() + 1,
14555                                    cell.coord.col() + 1,
14556                                    &empty,
14557                                );
14558                            }
14559                        }
14560                        self.graph.update_vertex_value_ref(vertex_id, &other);
14561                        // Optionally mirror into Arrow overlay for Arrow-backed reads
14562                        if self.config.arrow_storage_enabled
14563                            && self.config.delta_overlay_enabled
14564                            && self.config.write_formula_overlay_enabled
14565                        {
14566                            let anchor = self
14567                                .graph
14568                                .get_cell_ref(vertex_id)
14569                                .expect("cell ref for vertex");
14570                            let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14571                            self.mirror_value_to_computed_overlay(
14572                                &sheet_name,
14573                                anchor.coord.row() + 1,
14574                                anchor.coord.col() + 1,
14575                                &other,
14576                            );
14577                        }
14578                        Ok(other)
14579                    }
14580                }
14581            }
14582            Err(e) => {
14583                // Runtime Excel error: store as a cell value instead of propagating
14584                // as an exception so bulk eval paths don't fail the whole pass.
14585                let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14586                let err_val = LiteralValue::Error(e.clone());
14587                if let Some(d) = delta
14588                    && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14589                {
14590                    if spill_cells.is_empty() {
14591                        let old = self
14592                            .read_cell_value(
14593                                self.graph.sheet_name(anchor.sheet_id),
14594                                anchor.coord.row() + 1,
14595                                anchor.coord.col() + 1,
14596                            )
14597                            .unwrap_or(LiteralValue::Empty);
14598                        if old != err_val {
14599                            d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14600                        }
14601                    } else {
14602                        for cell in spill_cells.iter() {
14603                            let sheet_name = self.graph.sheet_name(cell.sheet_id);
14604                            let old = self
14605                                .read_cell_value(
14606                                    sheet_name,
14607                                    cell.coord.row() + 1,
14608                                    cell.coord.col() + 1,
14609                                )
14610                                .unwrap_or(LiteralValue::Empty);
14611                            let new = if cell.sheet_id == anchor.sheet_id
14612                                && cell.coord.row() == anchor.coord.row()
14613                                && cell.coord.col() == anchor.coord.col()
14614                            {
14615                                err_val.clone()
14616                            } else {
14617                                LiteralValue::Empty
14618                            };
14619                            Self::record_cell_if_changed(d, cell, &old, &new);
14620                        }
14621                    }
14622                }
14623                self.graph.clear_spill_region(vertex_id);
14624                if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14625                    self.record_structural_change(scope);
14626                }
14627                if self.config.arrow_storage_enabled
14628                    && self.config.delta_overlay_enabled
14629                    && self.config.write_formula_overlay_enabled
14630                {
14631                    let empty = LiteralValue::Empty;
14632                    for cell in spill_cells.iter() {
14633                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14634                        self.mirror_value_to_computed_overlay(
14635                            &sheet_name,
14636                            cell.coord.row() + 1,
14637                            cell.coord.col() + 1,
14638                            &empty,
14639                        );
14640                    }
14641                }
14642                self.graph.update_vertex_value_ref(vertex_id, &err_val);
14643                if self.config.arrow_storage_enabled
14644                    && self.config.delta_overlay_enabled
14645                    && self.config.write_formula_overlay_enabled
14646                {
14647                    let anchor = self
14648                        .graph
14649                        .get_cell_ref(vertex_id)
14650                        .expect("cell ref for vertex");
14651                    let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14652                    self.mirror_value_to_computed_overlay(
14653                        &sheet_name,
14654                        anchor.coord.row() + 1,
14655                        anchor.coord.col() + 1,
14656                        &err_val,
14657                    );
14658                }
14659                Ok(err_val)
14660            }
14661        }
14662    }
14663
14664    fn evaluate_named_scalar(
14665        &mut self,
14666        vertex_id: VertexId,
14667        sheet_id: SheetId,
14668    ) -> Result<LiteralValue, ExcelError> {
14669        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14670            ExcelError::new(ExcelErrorKind::Name)
14671                .with_message("Named range metadata missing".to_string())
14672        })?;
14673
14674        match &named_range.definition {
14675            NamedDefinition::Cell(cell_ref) => {
14676                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14677                let row = cell_ref.coord.row() + 1;
14678                let col = cell_ref.coord.col() + 1;
14679
14680                if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
14681                    && matches!(
14682                        self.graph.get_vertex_kind(dep_vertex),
14683                        VertexKind::FormulaScalar | VertexKind::FormulaArray
14684                    )
14685                {
14686                    // Graph does not cache cell/formula values; ensure the precedent is evaluated.
14687                    let value = self.evaluate_vertex(dep_vertex)?;
14688                    self.graph.update_vertex_value_ref(vertex_id, &value);
14689                    Ok(value)
14690                } else {
14691                    let value = self
14692                        .read_cell_value(sheet_name, row, col)
14693                        .unwrap_or(LiteralValue::Empty);
14694                    self.graph.update_vertex_value_ref(vertex_id, &value);
14695                    Ok(value)
14696                }
14697            }
14698            NamedDefinition::Literal(v) => {
14699                let out = v.clone();
14700                self.graph.update_vertex_value_ref(vertex_id, &out);
14701                Ok(out)
14702            }
14703            NamedDefinition::Formula { ast, .. } => {
14704                let context_sheet = match named_range.scope {
14705                    NameScope::Sheet(id) => id,
14706                    NameScope::Workbook => sheet_id,
14707                };
14708                let sheet_name = self.graph.sheet_name(context_sheet);
14709                let cell_ref = self
14710                    .graph
14711                    .get_cell_ref(vertex_id)
14712                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14713                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14714                    .with_resolved_functions(&self.resolved_functions)
14715                    .with_arena_cells(self);
14716                match interpreter.evaluate_ast(ast) {
14717                    Ok(cv) => {
14718                        let value = cv.into_literal();
14719                        match value {
14720                            LiteralValue::Array(_) => {
14721                                let err = ExcelError::new(ExcelErrorKind::NImpl)
14722                                    .with_message("Array result in scalar named range".to_string());
14723                                let err_val = LiteralValue::Error(err.clone());
14724                                self.graph.update_vertex_value_ref(vertex_id, &err_val);
14725                                Ok(err_val)
14726                            }
14727                            other => {
14728                                self.graph.update_vertex_value_ref(vertex_id, &other);
14729                                Ok(other)
14730                            }
14731                        }
14732                    }
14733                    Err(err) => {
14734                        let err_val = LiteralValue::Error(err.clone());
14735                        self.graph.update_vertex_value_ref(vertex_id, &err_val);
14736                        Ok(err_val)
14737                    }
14738                }
14739            }
14740            NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
14741                .with_message("Range-valued name evaluated as scalar".to_string())),
14742        }
14743    }
14744
14745    fn evaluate_named_array(
14746        &mut self,
14747        vertex_id: VertexId,
14748        sheet_id: SheetId,
14749    ) -> Result<LiteralValue, ExcelError> {
14750        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14751            ExcelError::new(ExcelErrorKind::Name)
14752                .with_message("Named range metadata missing".to_string())
14753        })?;
14754
14755        let out = match &named_range.definition {
14756            NamedDefinition::Range(range_ref) => {
14757                if range_ref.start.sheet_id != range_ref.end.sheet_id {
14758                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14759                        .with_message("Named range cannot span sheets".to_string()));
14760                }
14761
14762                let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
14763                let sr0 = range_ref.start.coord.row();
14764                let sc0 = range_ref.start.coord.col();
14765                let er0 = range_ref.end.coord.row();
14766                let ec0 = range_ref.end.coord.col();
14767                if sr0 > er0 || sc0 > ec0 {
14768                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14769                        .with_message("Invalid named range bounds".to_string()));
14770                }
14771
14772                let h = (er0 - sr0 + 1) as usize;
14773                let w = (ec0 - sc0 + 1) as usize;
14774                let cell_count = (h as u64).saturating_mul(w as u64);
14775                if cell_count > self.config.spill.max_spill_cells as u64 {
14776                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
14777                        "Named range too large to materialize as an array".to_string(),
14778                    ));
14779                }
14780
14781                let mut rows = Vec::with_capacity(h);
14782                for r0 in sr0..=er0 {
14783                    let mut row = Vec::with_capacity(w);
14784                    for c0 in sc0..=ec0 {
14785                        let v = self
14786                            .read_cell_value(sheet_name, r0 + 1, c0 + 1)
14787                            .unwrap_or(LiteralValue::Empty);
14788                        row.push(v);
14789                    }
14790                    rows.push(row);
14791                }
14792                LiteralValue::Array(rows)
14793            }
14794            NamedDefinition::Cell(cell_ref) => {
14795                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14796                let row = cell_ref.coord.row() + 1;
14797                let col = cell_ref.coord.col() + 1;
14798                let v = self
14799                    .read_cell_value(sheet_name, row, col)
14800                    .unwrap_or(LiteralValue::Empty);
14801                LiteralValue::Array(vec![vec![v]])
14802            }
14803            NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
14804            NamedDefinition::Formula { ast, .. } => {
14805                let context_sheet = match named_range.scope {
14806                    NameScope::Sheet(id) => id,
14807                    NameScope::Workbook => sheet_id,
14808                };
14809                let sheet_name = self.graph.sheet_name(context_sheet);
14810                let cell_ref = self
14811                    .graph
14812                    .get_cell_ref(vertex_id)
14813                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14814                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14815                    .with_resolved_functions(&self.resolved_functions)
14816                    .with_arena_cells(self);
14817                match interpreter.evaluate_ast(ast) {
14818                    Ok(cv) => {
14819                        let v = cv.into_literal();
14820                        match v {
14821                            LiteralValue::Array(_) => v,
14822                            other => LiteralValue::Array(vec![vec![other]]),
14823                        }
14824                    }
14825                    Err(err) => LiteralValue::Error(err),
14826                }
14827            }
14828        };
14829
14830        self.graph.update_vertex_value_ref(vertex_id, &out);
14831        Ok(out)
14832    }
14833
14834    fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
14835        crate::engine::ResourceLedgerError::Exhausted(
14836            formualizer_common::ResourceExhaustionDetail {
14837                reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
14838                limit: limit as u64,
14839                observed: limit.saturating_add(1) as u64,
14840                request_id: self
14841                    .active_evaluation_resource_request
14842                    .as_ref()
14843                    .map(|stats| stats.request_id),
14844            },
14845        )
14846        .into_excel_error()
14847        .with_message(format!("{context} did not converge after {limit} replans"))
14848    }
14849
14850    fn transient_target_preparation_stale(error: &ExcelError) -> bool {
14851        matches!(
14852            error.extra(),
14853            formualizer_common::ExcelErrorExtra::PreparationStale {
14854                reason: formualizer_common::PreparationStaleReason::Semantic
14855                    | formualizer_common::PreparationStaleReason::Provider
14856            }
14857        )
14858    }
14859
14860    fn prepare_graph_for_routed_evaluation(
14861        &mut self,
14862        targets: &[crate::engine::EvaluationTarget],
14863        options: &crate::engine::TargetEvalOptions<'_>,
14864    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
14865        const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
14866        let mut retries = 0usize;
14867        loop {
14868            match self.prepare_graph_for_targets_unobserved(targets, options) {
14869                Err(error)
14870                    if Self::transient_target_preparation_stale(&error)
14871                        && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
14872                {
14873                    retries = retries.saturating_add(1);
14874                }
14875                result => return result,
14876            }
14877        }
14878    }
14879
14880    fn evaluate_mixed_targets(
14881        &mut self,
14882        targets: &[crate::engine::EvaluationTarget],
14883        delta: Option<&mut DeltaCollector>,
14884    ) -> Result<EvalResult, ExcelError> {
14885        let _source_cache = self.source_cache_session();
14886        let cancel = self.active_cancel_flag.clone();
14887        let options = crate::engine::TargetEvalOptions {
14888            request_id: self
14889                .active_evaluation_resource_request
14890                .as_ref()
14891                .map(|stats| stats.request_id),
14892            cancel,
14893            deadline: None,
14894            budgets: None,
14895            opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
14896        };
14897        self.prepare_and_execute_target_recipe(targets, &options, delta)
14898    }
14899
14900    fn prepare_and_execute_target_recipe(
14901        &mut self,
14902        targets: &[crate::engine::EvaluationTarget],
14903        options: &crate::engine::TargetEvalOptions<'_>,
14904        delta: Option<&mut DeltaCollector>,
14905    ) -> Result<EvalResult, ExcelError> {
14906        let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
14907        self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
14908    }
14909
14910    fn execute_prepared_target_recipe(
14911        &mut self,
14912        targets: &[crate::engine::EvaluationTarget],
14913        scope: &crate::engine::PrepareScope,
14914        delta: Option<&mut DeltaCollector>,
14915    ) -> Result<EvalResult, ExcelError> {
14916        self.require_unified_authority()?;
14917        if matches!(scope, crate::engine::PrepareScope::Workbook)
14918            && let Some(stats) = self.active_evaluation_resource_request.as_mut()
14919        {
14920            stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
14921        }
14922        let mut roots = self.resolve_target_producers(targets)?;
14923        if let crate::engine::PrepareScope::Sheets(sheets) = scope {
14924            let request_id = self
14925                .active_evaluation_resource_request
14926                .as_ref()
14927                .map(|request| request.request_id);
14928            let root_count = roots.len();
14929            let mut widened_roots =
14930                OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
14931                    .map_err(|_| target_root_allocation_error(root_count, request_id))?;
14932            let sheet_ids = sheets
14933                .iter()
14934                .filter_map(|sheet| self.graph.sheet_id(sheet))
14935                .collect::<FxHashSet<_>>();
14936            for vertex in self.graph.formula_vertices() {
14937                if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
14938                    widened_roots
14939                        .push(crate::engine::target_preparation::TargetProducer::Legacy(
14940                            vertex,
14941                        ))
14942                        .map_err(|_| {
14943                            target_root_allocation_error(widened_roots.len() + 1, request_id)
14944                        })?;
14945                }
14946            }
14947            roots = widened_roots.into_vec();
14948        }
14949        self.begin_evaluation_request();
14950        #[cfg(any(test, feature = "legacy_oracle"))]
14951        self.graph.flush_pending_edge_deltas();
14952        let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
14953        if workbook_scope {
14954            if let Some(delta) = delta {
14955                self.evaluate_all_with_delta_collector(delta)
14956            } else {
14957                self.evaluate_all_legacy_impl()
14958            }
14959        } else {
14960            self.evaluate_legacy_target_roots(&roots, delta)
14961        }
14962    }
14963
14964    fn legacy_coordinate_targets(
14965        &mut self,
14966        targets: &[(&str, u32, u32)],
14967    ) -> Vec<crate::engine::EvaluationTarget> {
14968        targets
14969            .iter()
14970            .map(|(sheet, row, col)| {
14971                // Compatibility APIs historically interned an unknown target sheet
14972                // and returned an empty value rather than rejecting the target.
14973                self.graph.sheet_id_mut(sheet);
14974                crate::engine::EvaluationTarget::Cell {
14975                    sheet: (*sheet).to_string(),
14976                    row: *row,
14977                    col: *col,
14978                }
14979            })
14980            .collect()
14981    }
14982
14983    /// Evaluate the necessary mixed producer closure for typed cell, range, name, and table targets.
14984    pub fn evaluate_targets(
14985        &mut self,
14986        targets: &[crate::engine::EvaluationTarget],
14987    ) -> Result<EvalResult, ExcelError> {
14988        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14989            engine.observe_function_semantic_epoch()?;
14990            engine.validate_deterministic_mode()?;
14991            engine.evaluate_mixed_targets(targets, None)
14992        })
14993    }
14994
14995    /// Evaluate typed targets with explicit preparation policy and request controls.
14996    pub fn evaluate_targets_with_options(
14997        &mut self,
14998        targets: &[crate::engine::EvaluationTarget],
14999        options: crate::engine::TargetEvalOptions<'_>,
15000    ) -> Result<EvalResult, ExcelError> {
15001        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
15002            engine.active_cancel_flag = options.cancel.clone();
15003            engine.active_evaluation_deadline = options.deadline;
15004            let result = (|| {
15005                engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
15006                engine.observe_function_semantic_epoch()?;
15007                engine.validate_deterministic_mode()?;
15008                let _source_cache = engine.source_cache_session();
15009                engine.prepare_and_execute_target_recipe(targets, &options, None)
15010            })();
15011            engine.active_cancel_flag = None;
15012            engine.active_evaluation_deadline = None;
15013            result
15014        })
15015    }
15016
15017    /// Evaluate typed targets and return the versioned run/region delta for the request.
15018    pub fn evaluate_targets_with_delta(
15019        &mut self,
15020        targets: &[crate::engine::EvaluationTarget],
15021    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15022        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15023            engine.observe_function_semantic_epoch()?;
15024            engine.validate_deterministic_mode()?;
15025            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15026            let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
15027            Ok((result, collector.finish_target()))
15028        })
15029    }
15030
15031    /// Evaluate only the necessary precedents for specific target cells (demand-driven)
15032    pub fn evaluate_until(
15033        &mut self,
15034        targets: &[(&str, u32, u32)],
15035    ) -> Result<EvalResult, ExcelError> {
15036        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
15037            engine.evaluate_until_unobserved(targets)
15038        })
15039    }
15040
15041    fn evaluate_until_unobserved(
15042        &mut self,
15043        targets: &[(&str, u32, u32)],
15044    ) -> Result<EvalResult, ExcelError> {
15045        self.observe_function_semantic_epoch()?;
15046        let targets = self.legacy_coordinate_targets(targets);
15047        self.evaluate_mixed_targets(&targets, None)
15048    }
15049
15050    fn evaluate_until_with_delta_collector(
15051        &mut self,
15052        targets: &[(&str, u32, u32)],
15053        delta: &mut DeltaCollector,
15054    ) -> Result<EvalResult, ExcelError> {
15055        let targets = self.legacy_coordinate_targets(targets);
15056        self.evaluate_mixed_targets(&targets, Some(delta))
15057    }
15058
15059    fn evaluate_legacy_target_roots(
15060        &mut self,
15061        roots: &[crate::engine::target_preparation::TargetProducer],
15062        mut delta: Option<&mut DeltaCollector>,
15063    ) -> Result<EvalResult, ExcelError> {
15064        use crate::engine::target_preparation::TargetProducer;
15065        #[cfg(any(test, feature = "benchmark_internal"))]
15066        {
15067            self.recalc_reuse_probe
15068                .get_mut()
15069                .unwrap()
15070                .legacy_target_requests += 1;
15071        }
15072        let start = crate::instant::FzInstant::now();
15073        let root_vertices = roots
15074            .iter()
15075            .filter_map(|root| match root {
15076                TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
15077                TargetProducer::ValueOnly(_) => None,
15078            })
15079            .collect::<Vec<_>>();
15080        let mut computed_vertices = 0usize;
15081        let mut cycle_errors = 0usize;
15082        let mut replans = 0usize;
15083        const MAX_REPLAN: usize = 5;
15084        self.graph.authority_sync();
15085        loop {
15086            let (precedents_to_eval, old_vdeps) = self.demand_subgraph(&root_vertices)?;
15087            if precedents_to_eval.is_empty() {
15088                break;
15089            }
15090            #[cfg(any(test, feature = "benchmark_internal"))]
15091            {
15092                self.recalc_reuse_probe
15093                    .get_mut()
15094                    .unwrap()
15095                    .target_schedule_builds += 1;
15096            }
15097            let schedule = {
15098                self.graph.authority_sync();
15099                let mut ledger = self.active_resource_ledger.take();
15100                let result = self.create_authority_schedule(
15101                    &precedents_to_eval,
15102                    &old_vdeps,
15103                    ledger.as_mut(),
15104                );
15105                self.active_resource_ledger = ledger;
15106                result?
15107            };
15108            self.begin_pass(&schedule);
15109            for (unit_index, &unit) in schedule.units.iter().enumerate() {
15110                self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
15111                match unit {
15112                    ScheduleUnit::Cycle(index) => {
15113                        if self.handle_cycle_unit(
15114                            schedule.unit_cycle(index),
15115                            delta.as_deref_mut(),
15116                            None,
15117                            None,
15118                        )? > 0
15119                        {
15120                            cycle_errors = cycle_errors.saturating_add(1);
15121                        }
15122                    }
15123                    ScheduleUnit::Layer(index) => {
15124                        let layer = schedule.unit_layer(index);
15125                        let evaluated = if let Some(delta) = delta.as_deref_mut() {
15126                            if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15127                                self.evaluate_layer_parallel_with_delta(layer, delta)?
15128                            } else {
15129                                self.evaluate_layer_sequential_with_delta(layer, delta)?
15130                            }
15131                        } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15132                            self.evaluate_layer_parallel(layer)?
15133                        } else {
15134                            self.evaluate_layer_sequential(layer)?
15135                        };
15136                        computed_vertices = computed_vertices.saturating_add(evaluated);
15137                    }
15138                }
15139                if self.stop_after_unit(&schedule, unit_index) {
15140                    break;
15141                }
15142            }
15143            let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
15144            self.resource_checkpoint(0)?;
15145            if !self.finish_target_pass_dirty(&precedents_to_eval, &changed) {
15146                break;
15147            }
15148            if replans >= MAX_REPLAN {
15149                return Err(self.replan_exhausted_error(
15150                    MAX_REPLAN,
15151                    "targeted legacy dynamic dependency evaluation",
15152                ));
15153            }
15154            replans = replans.saturating_add(1);
15155        }
15156        self.redirty_for_next_recalc();
15157        Ok(EvalResult {
15158            computed_vertices,
15159            cycle_errors,
15160            elapsed: start.elapsed(),
15161        })
15162    }
15163
15164    /// Build a revision-bound compatibility plan covering every prepared formula vertex.
15165    pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
15166        if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
15167            return Err(
15168                Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
15169                    "compatibility recalculation plans require all staged formulas to be prepared",
15170                ),
15171            );
15172        }
15173        let key = self.recalc_plan_key();
15174        let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
15175        vertices.sort_unstable();
15176        let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
15177        let schedule = if vertices.is_empty() {
15178            crate::engine::Schedule {
15179                units: Vec::new(),
15180                layers: Vec::new(),
15181                cycles: Vec::new(),
15182            }
15183        } else {
15184            self.create_evaluation_schedule_uncached(&vertices, None)?.0
15185        };
15186        self.validate_recalc_plan_key(&key)?;
15187        Ok(RecalcPlan {
15188            key,
15189            kind: RecalcPlanKind::CompatibilityFull {
15190                schedule,
15191                has_dynamic_refs,
15192            },
15193        })
15194    }
15195
15196    /// Prepare stable typed targets and retain a revision-bound run-local recipe.
15197    pub fn build_recalc_plan_for_targets(
15198        &mut self,
15199        targets: &[crate::engine::EvaluationTarget],
15200    ) -> Result<RecalcPlan, ExcelError> {
15201        self.build_recalc_plan_for_targets_with_options(
15202            targets,
15203            crate::engine::TargetEvalOptions::default(),
15204        )
15205    }
15206
15207    pub fn build_recalc_plan_for_targets_with_options(
15208        &mut self,
15209        targets: &[crate::engine::EvaluationTarget],
15210        options: crate::engine::TargetEvalOptions<'_>,
15211    ) -> Result<RecalcPlan, ExcelError> {
15212        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15213            engine.observe_function_semantic_epoch()?;
15214            engine.validate_deterministic_mode()?;
15215            let _source_cache = engine.source_cache_session();
15216            let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
15217            #[cfg(any(test, feature = "legacy_oracle"))]
15218            engine.graph.flush_pending_edge_deltas();
15219            let topology = if matches!(
15220                preparation.widened_scope,
15221                crate::engine::PrepareScope::Workbook
15222            ) {
15223                RecalcTopology::Workbook
15224            } else {
15225                RecalcTopology::RunLocalRecipe
15226            };
15227            Ok(RecalcPlan {
15228                key: engine.recalc_plan_key(),
15229                kind: RecalcPlanKind::Target {
15230                    targets: targets.to_vec(),
15231                    scope: preparation.widened_scope,
15232                    topology,
15233                    dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
15234                },
15235            })
15236        })
15237    }
15238
15239    /// Evaluate using a previously constructed compatibility or target plan.
15240    pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
15241        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15242            engine.evaluate_recalc_plan_unobserved(plan)
15243        })
15244    }
15245
15246    pub fn evaluate_recalc_plan_with_controls(
15247        &mut self,
15248        plan: &RecalcPlan,
15249        cancel: Option<crate::engine::CancelToken>,
15250        deadline: Option<Instant>,
15251    ) -> Result<EvalResult, ExcelError> {
15252        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15253            engine.active_cancel_flag = cancel.clone();
15254            engine.active_evaluation_deadline = deadline;
15255            let result = engine.evaluate_recalc_plan_unobserved(plan);
15256            engine.active_cancel_flag = None;
15257            engine.active_evaluation_deadline = None;
15258            result
15259        })
15260    }
15261
15262    fn evaluate_recalc_plan_unobserved(
15263        &mut self,
15264        plan: &RecalcPlan,
15265    ) -> Result<EvalResult, ExcelError> {
15266        #[cfg(any(test, feature = "legacy_oracle"))]
15267        self.graph.flush_pending_edge_deltas();
15268        self.validate_recalc_plan_key(&plan.key)?;
15269        self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
15270        self.validate_deterministic_mode()?;
15271
15272        match &plan.kind {
15273            RecalcPlanKind::Target {
15274                targets,
15275                scope,
15276                topology,
15277                dynamic_policy,
15278            } => {
15279                debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
15280                debug_assert_eq!(
15281                    matches!(topology, RecalcTopology::Workbook),
15282                    matches!(scope, crate::engine::PrepareScope::Workbook)
15283                );
15284                let _source_cache = self.source_cache_session();
15285                self.execute_prepared_target_recipe(targets, scope, None)
15286            }
15287            RecalcPlanKind::CompatibilityFull {
15288                schedule,
15289                has_dynamic_refs,
15290            } => {
15291                let _source_cache = self.source_cache_session();
15292                self.begin_evaluation_request();
15293                if *has_dynamic_refs {
15294                    self.virtual_dep_fallback_activations =
15295                        self.virtual_dep_fallback_activations.saturating_add(1);
15296                    return self.evaluate_all_coordinator();
15297                }
15298
15299                let start = crate::instant::FzInstant::now();
15300                let dirty_vertices = self.graph.get_evaluation_vertices();
15301                if dirty_vertices.is_empty() {
15302                    return Ok(EvalResult {
15303                        computed_vertices: 0,
15304                        cycle_errors: 0,
15305                        elapsed: start.elapsed(),
15306                    });
15307                }
15308
15309                let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
15310                let mut computed_vertices = 0;
15311                let mut cycle_errors = 0;
15312                for &unit in &schedule.units {
15313                    self.cancellation_checkpoint(
15314                        "Evaluation cancelled before recalculation plan schedule unit",
15315                    )?;
15316                    match unit {
15317                        ScheduleUnit::Cycle(i) => {
15318                            let stamped = self.handle_cycle_unit(
15319                                schedule.unit_cycle(i),
15320                                None,
15321                                Some(&dirty_set),
15322                                None,
15323                            )?;
15324                            if stamped > 0 {
15325                                cycle_errors += 1;
15326                            }
15327                        }
15328                        ScheduleUnit::Layer(i) => {
15329                            let work: Vec<VertexId> = schedule
15330                                .unit_layer(i)
15331                                .vertices
15332                                .iter()
15333                                .copied()
15334                                .filter(|v| dirty_set.contains(v))
15335                                .collect();
15336                            if work.is_empty() {
15337                                continue;
15338                            }
15339                            let temp_layer = crate::engine::scheduler::Layer::new(work);
15340                            if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
15341                                computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
15342                            } else {
15343                                computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
15344                            }
15345                        }
15346                    }
15347                }
15348
15349                self.resource_checkpoint(0)?;
15350                self.graph.clear_dirty_flags(&dirty_vertices);
15351                self.redirty_for_next_recalc();
15352                Ok(EvalResult {
15353                    computed_vertices,
15354                    cycle_errors,
15355                    elapsed: start.elapsed(),
15356                })
15357            }
15358        }
15359    }
15360}
15361
15362impl<R> Engine<R>
15363where
15364    R: EvaluationContext,
15365{
15366    /// Refuse out-of-scope authority states before evaluation can demote spans
15367    /// or execute a legacy schedule. The public error type is unchanged; NImpl
15368    /// carries the exact internal Unsupported operation for the deferred-scope
15369    /// gate. Admission and allocation failures are not scope exceptions.
15370    fn require_unified_authority(&mut self) -> Result<(), ExcelError> {
15371        self.graph
15372            .authority()
15373            .map(|_| ())
15374            .map_err(Self::authority_excel_error)
15375    }
15376
15377    fn authority_excel_error(error: crate::engine::authority::store::AuthorityError) -> ExcelError {
15378        use crate::engine::authority::store::AuthorityError;
15379        // Some unchanged behavioral tests assert only `error.kind`, hiding
15380        // the operation in their panic. The opt-in gate trace proves which
15381        // typed error was actually returned; it never changes that error.
15382        #[cfg(test)]
15383        if std::env::var_os("FZ_AUTHORITY_DEFERRED_TRACE").is_some() {
15384            eprintln!("M1B_AUTHORITY_ERROR {error:?}");
15385        }
15386        let kind = match error {
15387            AuthorityError::Unsupported { .. } => ExcelErrorKind::NImpl,
15388            _ => ExcelErrorKind::Error,
15389        };
15390        ExcelError::new(kind).with_message(format!("unified_authority: {error:?}"))
15391    }
15392
15393    /// Evaluate all dirty/volatile vertices
15394    /// Evaluate a current result snapshot, optionally with cancellation.
15395    ///
15396    /// This evaluates exactly like [`Self::evaluate_all`] (or its cancellable
15397    /// counterpart), but leaves volatile vertices and their dependents Current
15398    /// at return rather than marking them dirty for the next cycle. It does
15399    /// not suppress iterative-SCC redirty or any other freshness check.
15400    ///
15401    /// Callers retaining the engine must start another evaluation request
15402    /// (`evaluate_all`, `evaluate_until`, `evaluate_cell`, or equivalent) to
15403    /// refresh volatile values. That request restores the deferred volatile
15404    /// redirty before selecting its work. Errors and cancellation never leave
15405    /// the snapshot mode active.
15406    pub fn evaluate_all_for_snapshot(
15407        &mut self,
15408        cancel: Option<crate::engine::CancelToken>,
15409    ) -> Result<EvalResult, ExcelError> {
15410        /// Leaves snapshot mode even when evaluation unwinds. A panic
15411        /// keeps the deferred volatile redirty pending, so the next request
15412        /// restores volatile work without touching the graph mid-unwind.
15413        struct SnapshotScope<'e, R: EvaluationContext>(&'e mut Engine<R>);
15414        impl<R: EvaluationContext> Drop for SnapshotScope<'_, R> {
15415            fn drop(&mut self) {
15416                self.0.snapshot_evaluation_active = false;
15417                if std::thread::panicking() {
15418                    self.0.snapshot_volatile_redirty_pending = true;
15419                }
15420            }
15421        }
15422        self.snapshot_evaluation_active = true;
15423        let result = {
15424            let scope = SnapshotScope(self);
15425            match cancel {
15426                Some(cancel) => scope.0.evaluate_all_cancellable(cancel),
15427                None => scope.0.evaluate_all(),
15428            }
15429        };
15430        if result.is_err() {
15431            self.snapshot_volatile_redirty_pending = false;
15432            self.graph.redirty_volatiles();
15433        }
15434        result
15435    }
15436
15437    /// Evaluate all dirty/volatile vertices.
15438    pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
15439        // `evaluate_all_unobserved` owns the `observe_function_semantic_epoch` guard.
15440        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
15441            engine.evaluate_all_unobserved()
15442        })
15443    }
15444
15445    fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
15446        debug_assert!(
15447            !self.graph.deferred_dirty_active(),
15448            "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
15449             was not balanced by end_deferred_dirty"
15450        );
15451        self.observe_function_semantic_epoch()?;
15452        self.lookup_index_cache.reset_counters();
15453        let _source_cache = self.source_cache_session();
15454        self.validate_deterministic_mode()?;
15455        if self.config.defer_graph_building {
15456            // Build graph for all staged formulas before evaluating
15457            self.build_graph_all()?;
15458        }
15459        self.evaluate_all_coordinator()
15460    }
15461
15462    /// Coordinator for `evaluate_all`: starts the evaluation request and runs
15463    /// the per-cell pass.
15464    fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
15465        self.require_unified_authority()?;
15466        self.begin_evaluation_request();
15467        self.evaluate_all_legacy_impl()
15468    }
15469
15470    /// Walk a schedule's units in condensation order: stamp each cyclic SCC
15471    /// at its position and evaluate each layer (parallel when enabled).
15472    ///
15473    /// Returns `(computed_vertices, cycle_count)` where `cycle_count` is the
15474    /// number of Cycle units walked (the former `schedule.cycles.len()`).
15475    fn legacy_pass_run_units(
15476        &mut self,
15477        schedule: &crate::engine::scheduler::Schedule,
15478    ) -> Result<(usize, usize), ExcelError> {
15479        let mut computed_vertices = 0;
15480        let mut cycle_count = 0;
15481        self.begin_pass(schedule);
15482        for (unit_index, &unit) in schedule.units.iter().enumerate() {
15483            match unit {
15484                ScheduleUnit::Cycle(i) => {
15485                    if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
15486                        cycle_count += 1;
15487                    }
15488                }
15489                ScheduleUnit::Layer(i) => {
15490                    let layer = schedule.unit_layer(i);
15491                    if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15492                        computed_vertices += self.evaluate_layer_parallel(layer)?;
15493                    } else {
15494                        computed_vertices += self.evaluate_layer_sequential(layer)?;
15495                    }
15496                }
15497            }
15498            if self.stop_after_unit(schedule, unit_index) {
15499                break;
15500            }
15501        }
15502        Ok((computed_vertices, cycle_count))
15503    }
15504
15505    /// Per-cell `evaluate_all` body, reached through the coordinator. This is
15506    /// an internal primitive; it must not be invoked directly from public APIs.
15507    ///
15508    /// Does NOT call `begin_evaluation_request` (cycle-telemetry reset +
15509    /// per-recalc clock sample): request begin happens at the public entry
15510    /// points / coordinators, so one request keeps one clock sample.
15511    fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
15512        self.reset_virtual_dep_telemetry_if_disabled();
15513        let _span_eval =
15514            crate::engine::trace::fz_span!(tracing::Level::INFO, "evaluate", "evaluate.legacy");
15515        let start = crate::instant::FzInstant::now();
15516        let mut computed_vertices = 0;
15517        let mut cycle_errors = 0;
15518        let mut replan_iterations = 0;
15519        const MAX_REPLAN: usize = 5;
15520        let mut telemetry = self
15521            .config
15522            .enable_virtual_dep_telemetry
15523            .then(|| self.start_virtual_dep_telemetry());
15524
15525        loop {
15526            let to_evaluate = self.graph.get_evaluation_vertices();
15527            if to_evaluate.is_empty() {
15528                if let Some(t) = telemetry.as_mut()
15529                    && t.bailout_reason.is_none()
15530                {
15531                    t.bailout_reason = Some("no_work");
15532                }
15533                break;
15534            }
15535
15536            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15537            if let Some(t) = telemetry.as_mut() {
15538                Self::accumulate_schedule_meta(t, &meta);
15539            }
15540
15541            let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
15542            computed_vertices += pass_computed;
15543            cycle_errors += pass_cycles;
15544
15545            // Check if dynamic dependencies changed
15546            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15547            if let Some(t) = telemetry.as_mut() {
15548                t.changed_vdeps_total += changed_vertices.len();
15549            }
15550
15551            self.resource_checkpoint(0)?;
15552            if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15553                if let Some(t) = telemetry.as_mut() {
15554                    t.bailout_reason = Some("converged");
15555                }
15556                break;
15557            }
15558            if replan_iterations >= MAX_REPLAN {
15559                if let Some(mut t) = telemetry.take() {
15560                    t.bailout_reason = Some("max_replan");
15561                    t.replan_iterations = replan_iterations;
15562                    self.last_virtual_dep_telemetry = t;
15563                }
15564                return Err(
15565                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15566                );
15567            }
15568
15569            replan_iterations += 1;
15570        }
15571
15572        if let Some(mut t) = telemetry {
15573            t.replan_iterations = replan_iterations;
15574            self.last_virtual_dep_telemetry = t;
15575        }
15576
15577        // Re-dirty volatile vertices for the next evaluation cycle
15578        self.redirty_for_next_recalc();
15579
15580        // Advance recalc epoch after a full evaluation pass finishes
15581        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15582
15583        Ok(EvalResult {
15584            computed_vertices,
15585            cycle_errors,
15586            elapsed: start.elapsed(),
15587        })
15588    }
15589
15590    pub fn evaluate_all_with_target_delta(
15591        &mut self,
15592    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15593        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15594            engine.observe_function_semantic_epoch()?;
15595            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15596            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15597            Ok((result, collector.finish_target()))
15598        })
15599    }
15600
15601    pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
15602        self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
15603    }
15604
15605    pub fn evaluate_all_with_delta_policy(
15606        &mut self,
15607        policy: EvalDeltaCompatibilityPolicy,
15608    ) -> Result<(EvalResult, EvalDelta), ExcelError> {
15609        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15610            engine.observe_function_semantic_epoch()?;
15611            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15612            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15613            Ok((result, collector.finish_with_policy(policy)?))
15614        })
15615    }
15616
15617    fn evaluate_all_with_delta_collector(
15618        &mut self,
15619        delta: &mut DeltaCollector,
15620    ) -> Result<EvalResult, ExcelError> {
15621        let _source_cache = self.source_cache_session();
15622        if self.config.defer_graph_building {
15623            self.build_graph_all()?;
15624        }
15625        self.require_unified_authority()?;
15626        self.begin_evaluation_request();
15627        self.reset_virtual_dep_telemetry_if_disabled();
15628        let _span_eval = crate::engine::trace::fz_span!(
15629            tracing::Level::INFO,
15630            "evaluate",
15631            "evaluate.legacy_delta"
15632        );
15633        let start = crate::instant::FzInstant::now();
15634        let mut computed_vertices = 0;
15635        let mut cycle_errors = 0;
15636
15637        let mut replan_iterations = 0;
15638        const MAX_REPLAN: usize = 5;
15639        let mut telemetry = self
15640            .config
15641            .enable_virtual_dep_telemetry
15642            .then(|| self.start_virtual_dep_telemetry());
15643
15644        loop {
15645            let to_evaluate = self.graph.get_evaluation_vertices();
15646            if to_evaluate.is_empty() {
15647                if let Some(t) = telemetry.as_mut()
15648                    && t.bailout_reason.is_none()
15649                {
15650                    t.bailout_reason = Some("no_work");
15651                }
15652                break;
15653            }
15654
15655            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15656            if let Some(t) = telemetry.as_mut() {
15657                Self::accumulate_schedule_meta(t, &meta);
15658            }
15659
15660            self.begin_pass(&schedule);
15661            for (unit_index, &unit) in schedule.units.iter().enumerate() {
15662                match unit {
15663                    ScheduleUnit::Cycle(i) => {
15664                        if self.handle_cycle_unit(
15665                            schedule.unit_cycle(i),
15666                            Some(delta),
15667                            None,
15668                            None,
15669                        )? > 0
15670                        {
15671                            cycle_errors += 1;
15672                        }
15673                    }
15674                    ScheduleUnit::Layer(i) => {
15675                        let layer = schedule.unit_layer(i);
15676                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15677                            computed_vertices +=
15678                                self.evaluate_layer_parallel_with_delta(layer, delta)?;
15679                        } else {
15680                            computed_vertices +=
15681                                self.evaluate_layer_sequential_with_delta(layer, delta)?;
15682                        }
15683                    }
15684                }
15685                if self.stop_after_unit(&schedule, unit_index) {
15686                    break;
15687                }
15688            }
15689
15690            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15691            if let Some(t) = telemetry.as_mut() {
15692                t.changed_vdeps_total += changed_vertices.len();
15693            }
15694            self.resource_checkpoint(0)?;
15695            if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15696                if let Some(t) = telemetry.as_mut() {
15697                    t.bailout_reason = Some("converged");
15698                }
15699                break;
15700            }
15701            if replan_iterations >= MAX_REPLAN {
15702                if let Some(mut t) = telemetry.take() {
15703                    t.bailout_reason = Some("max_replan");
15704                    t.replan_iterations = replan_iterations;
15705                    self.last_virtual_dep_telemetry = t;
15706                }
15707                return Err(
15708                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15709                );
15710            }
15711            replan_iterations += 1;
15712        }
15713
15714        if let Some(mut t) = telemetry {
15715            t.replan_iterations = replan_iterations;
15716            self.last_virtual_dep_telemetry = t;
15717        }
15718
15719        self.redirty_for_next_recalc();
15720        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15721
15722        Ok(EvalResult {
15723            computed_vertices,
15724            cycle_errors,
15725            elapsed: start.elapsed(),
15726        })
15727    }
15728
15729    /// Convenience: demand-driven evaluation of a single cell by sheet name and row/col.
15730    ///
15731    /// This will evaluate only the minimal set of dirty / volatile precedents required
15732    /// to bring the target cell up-to-date (as if a user asked for that single value),
15733    /// rather than scheduling a full workbook recalc. If the cell is already clean and
15734    /// non-volatile, no vertices will be recomputed.
15735    ///
15736    /// Returns the (possibly newly computed) value stored for the cell afterwards.
15737    /// Empty cells return None. Errors are surfaced via the Result type.
15738    pub fn evaluate_cell(
15739        &mut self,
15740        sheet: &str,
15741        row: u32,
15742        col: u32,
15743    ) -> Result<Option<LiteralValue>, ExcelError> {
15744        self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
15745            engine.evaluate_cell_unobserved(sheet, row, col)
15746        })
15747    }
15748
15749    fn evaluate_cell_unobserved(
15750        &mut self,
15751        sheet: &str,
15752        row: u32,
15753        col: u32,
15754    ) -> Result<Option<LiteralValue>, ExcelError> {
15755        if row == 0 || col == 0 {
15756            return Err(ExcelError::new(ExcelErrorKind::Ref)
15757                .with_message("Row and column must be >= 1".to_string()));
15758        }
15759
15760        let result = self.evaluate_cells(&[(sheet, row, col)])?;
15761
15762        match result.len() {
15763            0 => Ok(None),
15764            1 => {
15765                let v = result.into_iter().next().unwrap();
15766                Ok(v)
15767            }
15768            _ => unreachable!("evaluate_cells returned unexpected length"),
15769        }
15770    }
15771
15772    /// Convenience: demand-driven evaluation of multiple cells; accepts a slice of
15773    /// (sheet, row, col) triples. The union of required dirty / volatile precedents
15774    /// is computed once and evaluated, which is typically faster than calling
15775    /// `evaluate_cell` repeatedly for a related set of targets.
15776    ///
15777    /// Returns the resulting values for each requested target in the same order.
15778    pub fn evaluate_cells(
15779        &mut self,
15780        targets: &[(&str, u32, u32)],
15781    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15782        self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
15783            engine.evaluate_cells_unobserved(targets)
15784        })
15785    }
15786
15787    fn evaluate_cells_unobserved(
15788        &mut self,
15789        targets: &[(&str, u32, u32)],
15790    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15791        self.observe_function_semantic_epoch()?;
15792        debug_assert!(
15793            !self.graph.deferred_dirty_active(),
15794            "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
15795             was not balanced by end_deferred_dirty"
15796        );
15797        self.validate_deterministic_mode()?;
15798        if targets.is_empty() {
15799            return Ok(Vec::new());
15800        }
15801        let typed_targets = self.legacy_coordinate_targets(targets);
15802        self.evaluate_mixed_targets(&typed_targets, None)?;
15803        Ok(targets
15804            .iter()
15805            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15806            .collect())
15807    }
15808
15809    pub fn evaluate_cells_cancellable(
15810        &mut self,
15811        targets: &[(&str, u32, u32)],
15812        cancel: crate::engine::CancelToken,
15813    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15814        self.observe_evaluation_resource_request(
15815            EvaluationRequestKind::CellsCancellable,
15816            |engine| {
15817                engine.observe_function_semantic_epoch()?;
15818                engine.active_cancel_flag = Some(cancel.clone());
15819                let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
15820                engine.active_cancel_flag = None;
15821                res
15822            },
15823        )
15824    }
15825
15826    fn evaluate_cells_cancellable_impl(
15827        &mut self,
15828        targets: &[(&str, u32, u32)],
15829        cancel_flag: &AtomicBool,
15830    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15831        self.validate_deterministic_mode()?;
15832        if targets.is_empty() {
15833            return Ok(Vec::new());
15834        }
15835        if cancel_flag.load(Ordering::Relaxed) {
15836            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
15837                .with_message("Evaluation cancelled before target preparation"));
15838        }
15839        let typed_targets = self.legacy_coordinate_targets(targets);
15840        self.evaluate_mixed_targets(&typed_targets, None)?;
15841        Ok(targets
15842            .iter()
15843            .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
15844            .collect())
15845    }
15846
15847    pub fn evaluate_cells_with_target_delta(
15848        &mut self,
15849        targets: &[(&str, u32, u32)],
15850    ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
15851        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15852            engine.observe_function_semantic_epoch()?;
15853            engine.validate_deterministic_mode()?;
15854            if targets.is_empty() {
15855                return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
15856            }
15857            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15858            engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
15859            let values = targets
15860                .iter()
15861                .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
15862                .collect();
15863            Ok((values, collector.finish_target()))
15864        })
15865    }
15866
15867    pub fn evaluate_cells_with_delta(
15868        &mut self,
15869        targets: &[(&str, u32, u32)],
15870    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15871        self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
15872    }
15873
15874    pub fn evaluate_cells_with_delta_policy(
15875        &mut self,
15876        targets: &[(&str, u32, u32)],
15877        policy: EvalDeltaCompatibilityPolicy,
15878    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15879        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15880            engine.evaluate_cells_with_delta_unobserved(targets, policy)
15881        })
15882    }
15883
15884    fn evaluate_cells_with_delta_unobserved(
15885        &mut self,
15886        targets: &[(&str, u32, u32)],
15887        policy: EvalDeltaCompatibilityPolicy,
15888    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15889        self.observe_function_semantic_epoch()?;
15890        self.validate_deterministic_mode()?;
15891        if targets.is_empty() {
15892            return Ok((Vec::new(), EvalDelta::default()));
15893        }
15894        let mut collector = DeltaCollector::new(DeltaMode::Cells);
15895        self.evaluate_until_with_delta_collector(targets, &mut collector)?;
15896        let values = targets
15897            .iter()
15898            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15899            .collect();
15900        Ok((values, collector.finish_with_policy(policy)?))
15901    }
15902
15903    /// Get the evaluation plan for target cells without actually evaluating them
15904    pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
15905        if targets.is_empty() {
15906            return Ok(EvalPlan {
15907                total_vertices_to_evaluate: 0,
15908                layers: Vec::new(),
15909                cycles_detected: 0,
15910                dirty_count: 0,
15911                volatile_count: 0,
15912                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15913                estimated_parallel_layers: 0,
15914                target_cells: Vec::new(),
15915            });
15916        }
15917        if self.config.defer_graph_building && self.has_staged_formulas() {
15918            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
15919                "Evaluation plan requested with deferred graph; build first or call evaluate_*",
15920            ));
15921        }
15922
15923        // Convert targets to A1 notation for consistency
15924        let addresses: Vec<String> = targets
15925            .iter()
15926            .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
15927            .collect();
15928
15929        // Parse target cell addresses
15930        let mut target_addrs = Vec::new();
15931        for (sheet, row, col) in targets {
15932            if let Some(sheet_id) = self.graph.sheet_id(sheet) {
15933                let coord = Coord::from_excel(*row, *col, true, true);
15934                target_addrs.push(CellRef::new(sheet_id, coord));
15935            }
15936        }
15937
15938        // Find vertex IDs for targets
15939        let mut target_vertex_ids = Vec::new();
15940        for addr in &target_addrs {
15941            if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
15942                target_vertex_ids.push(vertex_id);
15943            }
15944        }
15945
15946        if target_vertex_ids.is_empty() {
15947            return Ok(EvalPlan {
15948                total_vertices_to_evaluate: 0,
15949                layers: Vec::new(),
15950                cycles_detected: 0,
15951                dirty_count: 0,
15952                volatile_count: 0,
15953                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15954                estimated_parallel_layers: 0,
15955                target_cells: addresses,
15956            });
15957        }
15958
15959        // Build demand subgraph with virtual edges (same as evaluate_until)
15960        let (precedents_to_eval, vdeps) = self.demand_subgraph(&target_vertex_ids)?;
15961
15962        if precedents_to_eval.is_empty() {
15963            return Ok(EvalPlan {
15964                total_vertices_to_evaluate: 0,
15965                layers: Vec::new(),
15966                cycles_detected: 0,
15967                dirty_count: 0,
15968                volatile_count: 0,
15969                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15970                estimated_parallel_layers: 0,
15971                target_cells: addresses,
15972            });
15973        }
15974
15975        // Count dirty and volatile vertices
15976        let mut dirty_count = 0;
15977        let mut volatile_count = 0;
15978        for &vertex_id in &precedents_to_eval {
15979            if self.graph.is_dirty(vertex_id) {
15980                dirty_count += 1;
15981            }
15982            if self.graph.is_volatile(vertex_id) {
15983                volatile_count += 1;
15984            }
15985        }
15986
15987        // Create schedule for the minimal subgraph honoring virtual edges
15988        let schedule = self.create_authority_schedule(&precedents_to_eval, &vdeps, None)?;
15989
15990        // Build layer information
15991        let mut layers = Vec::new();
15992        let mut estimated_parallel_layers = 0;
15993        let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
15994
15995        for layer in &schedule.layers {
15996            let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
15997            if parallel_eligible {
15998                estimated_parallel_layers += 1;
15999            }
16000
16001            // Get sample cell addresses (up to 5)
16002            let sample_cells: Vec<String> = layer
16003                .vertices
16004                .iter()
16005                .take(5)
16006                .filter_map(|&vertex_id| {
16007                    self.graph
16008                        .get_cell_ref_for_vertex(vertex_id)
16009                        .map(|cell_ref| {
16010                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
16011                            format!(
16012                                "{}!{}{}",
16013                                sheet_name,
16014                                Self::col_to_letters(cell_ref.coord.col().saturating_add(1)),
16015                                cell_ref.coord.row() + 1
16016                            )
16017                        })
16018                })
16019                .collect();
16020
16021            layers.push(LayerInfo {
16022                vertex_count: layer.vertices.len(),
16023                parallel_eligible,
16024                sample_cells,
16025            });
16026        }
16027
16028        Ok(EvalPlan {
16029            total_vertices_to_evaluate: precedents_to_eval.len(),
16030            layers,
16031            cycles_detected: schedule.cycles.len(),
16032            dirty_count,
16033            volatile_count,
16034            parallel_enabled,
16035            estimated_parallel_layers,
16036            target_cells: addresses,
16037        })
16038    }
16039    /// Helper to create a schedule, integrating virtual dependencies automatically.
16040    fn create_evaluation_schedule(
16041        &mut self,
16042        to_evaluate: &[VertexId],
16043    ) -> Result<EvaluationScheduleBuildOutput, ExcelError> {
16044        #[cfg(any(test, feature = "benchmark_internal"))]
16045        {
16046            self.recalc_reuse_probe.get_mut().unwrap().schedule_requests += 1;
16047        }
16048        // Fold pending edge deltas once per schedule build so traversal uses
16049        // the zero-allocation CSR slices (#125).
16050        #[cfg(any(test, feature = "legacy_oracle"))]
16051        self.graph.flush_pending_edge_deltas();
16052        // The cache key includes the authority revision: sync first.
16053        self.graph.authority_sync();
16054        if self.can_use_static_schedule_cache(to_evaluate) {
16055            // A recent schedule for the same request becomes the current one.
16056            let revision = self.schedule_cache_authority_revision();
16057            let current = |e: &CachedScheduleEntry| {
16058                e.topology_epoch == self.topology_epoch && e.authority_revision == revision
16059            };
16060            if !self
16061                .cached_static_schedule
16062                .as_ref()
16063                .is_some_and(|c| current(c) && c.candidate_vertices.equals(to_evaluate))
16064                && let Some(i) = self.recent_schedules.iter().position(|e| {
16065                    current(e)
16066                        && e.candidate_vertices.len() == to_evaluate.len()
16067                        && e.candidate_vertices.equals(to_evaluate)
16068                })
16069            {
16070                let hit = self.recent_schedules.remove(i);
16071                if let Some(previous) = self.cached_static_schedule.replace(hit) {
16072                    self.retain_recent_schedule(previous);
16073                }
16074            }
16075            if let Some(cached) = self.cached_static_schedule.as_ref()
16076                && cached.topology_epoch == self.topology_epoch
16077                && cached.authority_revision == self.schedule_cache_authority_revision()
16078                && cached.candidate_vertices.equals(to_evaluate)
16079            {
16080                let meta = ScheduleBuildMeta {
16081                    candidate_vertices: to_evaluate.len(),
16082                    vdeps_vertices: 0,
16083                    vdeps_edges: 0,
16084                    builder_elapsed_ms: 0,
16085                    used_virtual_schedule: false,
16086                    schedule_cache_hit: true,
16087                    schedule_cache_eligible: true,
16088                };
16089                #[cfg(any(test, feature = "benchmark_internal"))]
16090                {
16091                    let mut probe = self.recalc_reuse_probe.lock().unwrap();
16092                    probe.schedule_cache_hits += 1;
16093                    probe.schedule_shared_handles += 1;
16094                }
16095                return Ok((
16096                    EvaluationSchedule::Shared(Arc::clone(&cached.schedule)),
16097                    FxHashMap::default(),
16098                    meta,
16099                ));
16100            }
16101
16102            let (schedule, vdeps, mut meta) = match self.schedule_from_base(to_evaluate)? {
16103                Some(schedule) => (
16104                    schedule,
16105                    FxHashMap::default(),
16106                    ScheduleBuildMeta {
16107                        candidate_vertices: to_evaluate.len(),
16108                        vdeps_vertices: 0,
16109                        vdeps_edges: 0,
16110                        builder_elapsed_ms: 0,
16111                        used_virtual_schedule: false,
16112                        schedule_cache_hit: false,
16113                        schedule_cache_eligible: true,
16114                    },
16115                ),
16116                None => self.create_evaluation_schedule_active(to_evaluate)?,
16117            };
16118            meta.schedule_cache_hit = false;
16119            meta.schedule_cache_eligible = true;
16120            #[cfg(any(test, feature = "benchmark_internal"))]
16121            {
16122                self.recalc_reuse_probe
16123                    .get_mut()
16124                    .unwrap()
16125                    .schedule_cache_misses += 1;
16126            }
16127            let schedule = if vdeps.is_empty() {
16128                // Clone previously discarded builder spare capacity. Keep that compact
16129                // retained payload while sharing it with the current request.
16130                let mut schedule = schedule;
16131                schedule.units.shrink_to_fit();
16132                for layer in &mut schedule.layers {
16133                    layer.vertices.shrink_to_fit();
16134                }
16135                schedule.layers.shrink_to_fit();
16136                for cycle in &mut schedule.cycles {
16137                    cycle.shrink_to_fit();
16138                }
16139                schedule.cycles.shrink_to_fit();
16140                let schedule = Arc::new(schedule);
16141                #[cfg(any(test, feature = "benchmark_internal"))]
16142                {
16143                    self.recalc_reuse_probe
16144                        .get_mut()
16145                        .unwrap()
16146                        .schedule_shared_handles += 1;
16147                }
16148                let entry = CachedScheduleEntry {
16149                    topology_epoch: self.topology_epoch,
16150                    authority_revision: self.schedule_cache_authority_revision(),
16151                    candidate_vertices: VertexIdRuns::from_slice(to_evaluate),
16152                    schedule: Arc::clone(&schedule),
16153                };
16154                if let Some(previous) = self.cached_static_schedule.replace(entry) {
16155                    self.retain_recent_schedule(previous);
16156                }
16157                EvaluationSchedule::Shared(schedule)
16158            } else {
16159                EvaluationSchedule::Owned(schedule)
16160            };
16161            return Ok((schedule, vdeps, meta));
16162        }
16163
16164        let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_active(to_evaluate)?;
16165        meta.schedule_cache_hit = false;
16166        meta.schedule_cache_eligible = false;
16167        #[cfg(any(test, feature = "benchmark_internal"))]
16168        {
16169            self.recalc_reuse_probe
16170                .get_mut()
16171                .unwrap()
16172                .schedule_cache_ineligible += 1;
16173        }
16174        Ok((EvaluationSchedule::Owned(schedule), vdeps, meta))
16175    }
16176
16177    /// Plan reuse: the base schedule restricted to `to_evaluate` when it
16178    /// is current, covers the request, has at least
16179    /// `BASE_SCHEDULE_MIN_REQUEST` candidates, and is at most
16180    /// `BASE_SCHEDULE_RATIO` times its size (restricting walks the whole
16181    /// base; planning costs far more per candidate).
16182    fn schedule_from_base(
16183        &mut self,
16184        to_evaluate: &[VertexId],
16185    ) -> Result<Option<crate::engine::scheduler::Schedule>, ExcelError> {
16186        let revision = self.schedule_cache_authority_revision();
16187        let current = |e: &&CachedScheduleEntry| {
16188            e.topology_epoch == self.topology_epoch && e.authority_revision == revision
16189        };
16190        // The base, or the current schedule when larger (the first
16191        // evaluation's, before a later request replaces it).
16192        let Some(base) = [
16193            self.base_schedule.as_ref(),
16194            self.cached_static_schedule.as_ref(),
16195        ]
16196        .into_iter()
16197        .flatten()
16198        .filter(current)
16199        .max_by_key(|e| e.candidate_vertices.len()) else {
16200            return Ok(None);
16201        };
16202        let base_len = base.candidate_vertices.len();
16203        if base_len <= BASE_SCHEDULE_MIN_VERTICES
16204            || to_evaluate.len() < BASE_SCHEDULE_MIN_REQUEST
16205            || to_evaluate.len() > base_len
16206            || base_len > to_evaluate.len().saturating_mul(BASE_SCHEDULE_RATIO)
16207        {
16208            return Ok(None);
16209        }
16210        let mut keep = crate::engine::idset::DenseIdSet::default();
16211        keep.extend(to_evaluate.iter().copied());
16212        let Some((schedule, kept)) = base.schedule.restrict(&keep) else {
16213            return Ok(None);
16214        };
16215        // Every requested vertex must be in the base.
16216        if kept != keep.len() {
16217            return Ok(None);
16218        }
16219        if let Some(ledger) = self.active_resource_ledger.as_mut() {
16220            let layers: usize = schedule
16221                .layers
16222                .iter()
16223                .map(|l| {
16224                    l.vertices.capacity() * std::mem::size_of::<VertexId>()
16225                        + l.runs.capacity()
16226                            * std::mem::size_of::<crate::engine::scheduler::LayerRun>()
16227                })
16228                .sum();
16229            let bytes = (keep.heap_bytes()
16230                + layers
16231                + schedule.layers.capacity()
16232                    * std::mem::size_of::<crate::engine::scheduler::Layer>()
16233                + schedule.units.capacity()
16234                    * std::mem::size_of::<crate::engine::scheduler::ScheduleUnit>()
16235                + schedule
16236                    .cycles
16237                    .iter()
16238                    .map(|c| c.capacity() * 4)
16239                    .sum::<usize>()) as u64;
16240            ledger
16241                .reserve_schedule_discovery(bytes)
16242                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16243            ledger
16244                .release_scratch(bytes)
16245                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16246        }
16247        #[cfg(debug_assertions)]
16248        self.debug_check_restricted_schedule(to_evaluate, &schedule);
16249        #[cfg(any(test, feature = "benchmark_internal"))]
16250        {
16251            self.recalc_reuse_probe
16252                .get_mut()
16253                .unwrap()
16254                .schedule_base_restrictions += 1;
16255        }
16256        Ok(Some(schedule))
16257    }
16258
16259    /// Debug builds: a restricted schedule holds each requested vertex
16260    /// once and orders the request like a freshly planned schedule (every
16261    /// dependency among the request in an earlier unit).
16262    #[cfg(debug_assertions)]
16263    fn debug_check_restricted_schedule(
16264        &self,
16265        to_evaluate: &[VertexId],
16266        schedule: &crate::engine::scheduler::Schedule,
16267    ) {
16268        let mut position: FxHashMap<VertexId, usize> = FxHashMap::default();
16269        for (u, unit) in schedule.units.iter().enumerate() {
16270            let vertices: &[VertexId] = match *unit {
16271                crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16272                    &schedule.unit_layer(i).vertices
16273                }
16274                crate::engine::scheduler::ScheduleUnit::Cycle(i) => schedule.unit_cycle(i),
16275            };
16276            for &v in vertices {
16277                assert!(
16278                    position.insert(v, u).is_none(),
16279                    "vertex twice in a restricted schedule"
16280                );
16281            }
16282        }
16283        assert_eq!(position.len(), to_evaluate.len());
16284        if to_evaluate.len() > 512 {
16285            return;
16286        }
16287        // Every arc among the request (the store's edge images) goes to an
16288        // earlier unit, or within one sequential layer (a chain) or cycle.
16289        let Ok(store) = self.graph.authority_plan_store() else {
16290            return;
16291        };
16292        let cells: Vec<(VertexId, (u16, u32, u32))> = to_evaluate
16293            .iter()
16294            .filter_map(|&v| self.graph.authority_cell_of_vertex(v).map(|c| (v, c)))
16295            .collect();
16296        for &(v, (sheet, row, col)) in &cells {
16297            let Some(owner) = store.owner_at((sheet, row, col)) else {
16298                continue;
16299            };
16300            let Ok(refined) = store.refine_owner_column(owner, col, row, row, None) else {
16301                continue;
16302            };
16303            for piece in &refined.pieces {
16304                for edge in &refined.edges[piece.edge_start..piece.edge_end] {
16305                    let Some(image) = edge.proj.forward(&piece.domain) else {
16306                        continue;
16307                    };
16308                    for &(d, (s2, r2, c2)) in &cells {
16309                        if d == v
16310                            || edge.proj.sheet != s2
16311                            || !(image.r0 <= r2
16312                                && r2 <= image.r1
16313                                && image.c0 <= c2
16314                                && c2 <= image.c1)
16315                        {
16316                            continue;
16317                        }
16318                        let (pd, pv) = (position[&d], position[&v]);
16319                        let same_ok = pd == pv
16320                            && match schedule.units[pv] {
16321                                crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16322                                    schedule.unit_layer(i).sequential
16323                                }
16324                                crate::engine::scheduler::ScheduleUnit::Cycle(_) => true,
16325                            };
16326                        assert!(
16327                            pd < pv || same_ok,
16328                            "restricted schedule orders {v:?} before its precedent {d:?}"
16329                        );
16330                    }
16331                }
16332            }
16333        }
16334    }
16335
16336    /// Compress family formulas once per authority build (see
16337    /// `EvalConfig::formula_compression`), when no staged or deferred
16338    /// formula package can hold arena ids.
16339    fn maybe_compress_formulas(&mut self) {
16340        if !self.config.formula_compression {
16341            return;
16342        }
16343        let builds = self.graph.authority_host().builds;
16344        if self.compressed_at_build == Some(builds) {
16345            return;
16346        }
16347        self.compressed_at_build = Some(builds);
16348        if self.has_staged_formulas() {
16349            // Staged packages hold arena ids: no compaction. Members that
16350            // are already compressed can still leave the per-cell maps.
16351            self.graph.virtualize_family_members();
16352            return;
16353        }
16354        let pool = self.thread_pool.clone();
16355        let (_, garbage) = self.graph.compress_family_formulas(pool.as_deref());
16356        self.graph.virtualize_family_members();
16357        // Freeing the dropped members' reference texts (one allocation each)
16358        // is most of compaction; with a pool it happens off the critical
16359        // path.
16360        match pool {
16361            Some(pool) if garbage.len() >= 1024 => pool.spawn(move || drop(garbage)),
16362            _ => drop(garbage),
16363        }
16364    }
16365
16366    fn create_evaluation_schedule_active(
16367        &mut self,
16368        to_evaluate: &[VertexId],
16369    ) -> Result<ScheduleBuildOutput, ExcelError> {
16370        self.graph.authority_sync();
16371        self.maybe_compress_formulas();
16372        let mut ledger = self.active_resource_ledger.take();
16373        let result = self.create_evaluation_schedule_uncached(to_evaluate, ledger.as_mut());
16374        self.active_resource_ledger = ledger;
16375        result
16376    }
16377
16378    fn create_evaluation_schedule_uncached(
16379        &self,
16380        to_evaluate: &[VertexId],
16381        #[allow(unused_variables)] ledger: Option<&mut ResourceLedger>,
16382    ) -> Result<ScheduleBuildOutput, ExcelError> {
16383        #[cfg(any(test, feature = "benchmark_internal"))]
16384        {
16385            self.recalc_reuse_probe.lock().unwrap().schedule_builds += 1;
16386        }
16387        let builder = VirtualDepBuilder::new(self);
16388        #[allow(unused_mut)]
16389        let (mut vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
16390            if self.config.enable_virtual_dep_telemetry {
16391                let build_started = crate::instant::FzInstant::now();
16392                let (vdeps, augmented) = builder.build(to_evaluate);
16393                let builder_elapsed_ms = build_started.elapsed().as_millis();
16394                let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
16395                (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
16396            } else {
16397                let (vdeps, augmented) = builder.build(to_evaluate);
16398                (vdeps, augmented, 0, 0)
16399            };
16400
16401        // Replan hints from stale dynamic reads earlier in this request.
16402        {
16403            self.freshness_merge_hints(to_evaluate, &mut vdeps);
16404            self.freshness_extent_hints(to_evaluate, &mut vdeps);
16405        }
16406        let mut final_evaluate = to_evaluate.to_vec();
16407        if !augmented.is_empty() {
16408            final_evaluate.extend(augmented);
16409            final_evaluate.sort_unstable();
16410            final_evaluate.dedup();
16411        }
16412
16413        let use_virtual = !vdeps.is_empty();
16414
16415        let schedule = self.create_authority_schedule(&final_evaluate, &vdeps, ledger)?;
16416
16417        let meta = ScheduleBuildMeta {
16418            candidate_vertices: to_evaluate.len(),
16419            vdeps_vertices: vdeps.len(),
16420            vdeps_edges,
16421            builder_elapsed_ms,
16422            used_virtual_schedule: use_virtual,
16423            schedule_cache_hit: false,
16424            schedule_cache_eligible: false,
16425        };
16426
16427        Ok((schedule, vdeps, meta))
16428    }
16429
16430    fn create_authority_schedule(
16431        &self,
16432        candidates: &[VertexId],
16433        vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16434        mut ledger: Option<&mut ResourceLedger>,
16435    ) -> Result<crate::engine::scheduler::Schedule, ExcelError> {
16436        use crate::engine::authority::{
16437            geom::{Cover, Rect},
16438            plan_schedule, planner,
16439            proj::{AxisMap, RefProj},
16440            store::{EdgeKey, Tag},
16441        };
16442        let failure = |message: String| {
16443            ExcelError::new(ExcelErrorKind::Error)
16444                .with_message(format!("unified_authority planner: {message}"))
16445        };
16446        self.cancellation_checkpoint("Evaluation cancelled before authority planning")?;
16447        let store = self
16448            .graph
16449            .authority_plan_store()
16450            .map_err(Self::authority_excel_error)?;
16451        // One formula cell without hints (a tiny edit): its plan is the cell
16452        // alone unless it reads itself (`planner::plan_single`).
16453        if let [only] = candidates
16454            && vdeps.is_empty()
16455            && self.graph.authority_host().observed(*only).is_none()
16456            && let Some(cell) = self.graph.authority_cell_of_vertex(*only)
16457            && cell.0 != crate::engine::authority::geom::SYMBOL_SHEET
16458            && let Some(single) = planner::plan_single(store, cell)
16459        {
16460            #[cfg(debug_assertions)]
16461            {
16462                let mut cover = Cover::new();
16463                cover.insert_rect(cell.0, &Rect::new(cell.1, cell.2, cell.1, cell.2));
16464                let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16465                    .expect("general plan of one cell");
16466                assert_eq!(
16467                    general.cells.as_slice(),
16468                    &[single],
16469                    "single-cell plan differs from the planner at {cell:?}"
16470                );
16471            }
16472            let adapted = plan_schedule::schedule(
16473                &[single],
16474                0,
16475                None,
16476                |cell| {
16477                    self.graph
16478                        .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16479                        .ok_or_else(|| failure("missing executor identity".to_owned()))
16480                },
16481                |_work| Ok(()),
16482            )
16483            .map_err(|error| match error {
16484                plan_schedule::ScheduleError::Runtime(error) => error,
16485                other => failure(format!("{other:?}")),
16486            })?;
16487            if let Some(ledger) = ledger {
16488                ledger
16489                    .reserve_schedule_discovery(adapted.peak_heap_bytes)
16490                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16491                ledger
16492                    .release_scratch(adapted.peak_heap_bytes)
16493                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16494            }
16495            return Ok(adapted.schedule);
16496        }
16497        // A small request of grid formula cells without hints: per-cell
16498        // arcs and longest-path levels (`planner::plan_small`).
16499        if (2..=planner::SMALL_PLAN_MAX).contains(&candidates.len())
16500            && vdeps.is_empty()
16501            && candidates
16502                .iter()
16503                .all(|&v| self.graph.authority_host().observed(v).is_none())
16504        {
16505            let cells: Option<Vec<(u16, u32, u32)>> = candidates
16506                .iter()
16507                .map(|&v| self.graph.authority_cell_of_vertex(v))
16508                .collect();
16509            if let Some(small) = cells.as_deref().and_then(|c| planner::plan_small(store, c)) {
16510                #[cfg(debug_assertions)]
16511                {
16512                    let mut cover = Cover::new();
16513                    for c in cells.as_deref().unwrap_or_default() {
16514                        cover.insert_rect(c.0, &Rect::new(c.1, c.2, c.1, c.2));
16515                    }
16516                    let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16517                        .expect("general plan of a small request");
16518                    let key = |c: &planner::OrderedCell| (c.sheet, c.row, c.col, c.id, c.owner);
16519                    let mut a: Vec<_> = general.cells.iter().map(key).collect();
16520                    let mut b: Vec<_> = small.iter().map(key).collect();
16521                    a.sort_unstable();
16522                    b.sort_unstable();
16523                    assert_eq!(a, b, "small plan cells differ from the planner");
16524                    assert!(
16525                        general.cells.iter().all(|c| c.cycle.is_none()),
16526                        "small plan of a cyclic request"
16527                    );
16528                }
16529                let adapted = plan_schedule::schedule(
16530                    &small,
16531                    0,
16532                    None,
16533                    |cell| {
16534                        self.graph
16535                            .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16536                            .ok_or_else(|| failure("missing executor identity".to_owned()))
16537                    },
16538                    |_work| Ok(()),
16539                )
16540                .map_err(|error| match error {
16541                    plan_schedule::ScheduleError::Runtime(error) => error,
16542                    other => failure(format!("{other:?}")),
16543                })?;
16544                if let Some(ledger) = ledger {
16545                    ledger
16546                        .reserve_schedule_discovery(adapted.peak_heap_bytes)
16547                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16548                    ledger
16549                        .release_scratch(adapted.peak_heap_bytes)
16550                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16551                }
16552                return Ok(adapted.schedule);
16553            }
16554        }
16555        // Names are symbol-plane nodes (design §4.1): a name vertex plans as
16556        // the unit at its node, between its precedents and its readers.
16557        // Candidates become cells, sorted by (sheet, column, row) and
16558        // coalesced into row intervals: one cover insert per interval.
16559        let mut cover = Cover::new();
16560        {
16561            let cell_of = |&id: &VertexId| {
16562                self.graph
16563                    .authority_cell_of_vertex(id)
16564                    .map(|(sheet, row, col)| (sheet, col, row))
16565            };
16566            // A full recalc maps and sorts every formula: on the pool when
16567            // there is one (first eval's schedule is serial work otherwise).
16568            let cells: Vec<(u16, u32, u32)> = match self.thread_pool.as_deref() {
16569                Some(pool) if candidates.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16570                    use rayon::prelude::*;
16571                    pool.install(|| {
16572                        let mut cells: Vec<_> = candidates.par_iter().filter_map(cell_of).collect();
16573                        cells.par_sort_unstable();
16574                        cells
16575                    })
16576                }
16577                _ => {
16578                    let mut cells: Vec<_> = candidates.iter().filter_map(cell_of).collect();
16579                    cells.sort_unstable();
16580                    cells
16581                }
16582            };
16583            let mut i = 0;
16584            while i < cells.len() {
16585                let (sheet, col, r0) = cells[i];
16586                let mut r1 = r0;
16587                let mut j = i + 1;
16588                while j < cells.len() && cells[j].0 == sheet && cells[j].1 == col {
16589                    if cells[j].2 > r1 + 1 {
16590                        break;
16591                    }
16592                    r1 = r1.max(cells[j].2);
16593                    j += 1;
16594                }
16595                cover.insert_rect(sheet, &Rect::new(r0, col, r1, col));
16596                i = j;
16597            }
16598        }
16599        let mut hints = Vec::new();
16600        for (&reader, deps) in vdeps {
16601            let Some(reader) = self.graph.authority_cell_of_vertex(reader) else {
16602                continue;
16603            };
16604            for &dependency in deps {
16605                let Some(dep) = self.graph.authority_cell_of_vertex(dependency) else {
16606                    continue;
16607                };
16608                hints.push(planner::PlanHint {
16609                    reader: (reader.0, reader.2, reader.1),
16610                    edge: EdgeKey {
16611                        dep_sheet: reader.0,
16612                        tag: Tag::X,
16613                        lk: u32::MAX,
16614                        proj: RefProj {
16615                            sheet: dep.0,
16616                            rows: AxisMap::fixed(dep.1, dep.1),
16617                            cols: AxisMap::fixed(dep.2, dep.2),
16618                        },
16619                    },
16620                });
16621            }
16622        }
16623        // rdi_dyn: order each dynamic reader after its observed reads.
16624        let host = self.graph.authority_host();
16625        for &id in candidates.iter().filter(|_| host.has_observed()) {
16626            let Some(reads) = host.observed(id) else {
16627                continue;
16628            };
16629            let Some(reader) = self.graph.authority_cell_of_vertex(id) else {
16630                continue;
16631            };
16632            for &(sheet, r0, c0, r1, c1) in reads {
16633                hints.push(planner::PlanHint {
16634                    reader: (reader.0, reader.2, reader.1),
16635                    edge: EdgeKey {
16636                        dep_sheet: reader.0,
16637                        tag: Tag::X,
16638                        lk: u32::MAX,
16639                        proj: RefProj {
16640                            sheet,
16641                            rows: AxisMap::fixed(r0, r1),
16642                            cols: AxisMap::fixed(c0, c1),
16643                        },
16644                    },
16645                });
16646            }
16647        }
16648        hints.sort_unstable_by_key(|hint| hint.reader);
16649        let checkpoint = ledger
16650            .as_ref()
16651            .map_or(0, |ledger| ledger.scratch_checkpoint());
16652        let scratch_limit = ledger
16653            .as_ref()
16654            .and_then(|ledger| ledger.schedule_discovery_limit())
16655            .map(|limit| limit.saturating_sub(checkpoint));
16656        let ordered = planner::plan_with_hints(store, &cover, &hints, scratch_limit, None, None)
16657            .map_err(|error| failure(format!("{error:?}")))?;
16658        // max_work_units is an execution budget. Planning work must be capped
16659        // independently: charging it here changes the observable publication
16660        // boundary (e.g. a spill must commit before the next execution fails).
16661        let adapted = plan_schedule::schedule(
16662            &ordered.cells,
16663            ordered.heap_bytes(),
16664            scratch_limit,
16665            |cell| {
16666                self.graph
16667                    .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16668                    .ok_or_else(|| failure("missing executor identity".to_owned()))
16669            },
16670            |_work| {
16671                self.cancellation_checkpoint("Evaluation cancelled during authority planning")?;
16672                if let Some(ledger) = ledger.as_deref_mut() {
16673                    ledger
16674                        .checkpoint_deadline()
16675                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16676                }
16677                Ok(())
16678            },
16679        )
16680        .map_err(|error| match error {
16681            plan_schedule::ScheduleError::Runtime(error) => error,
16682            other => failure(format!("{other:?}")),
16683        })?;
16684        if let Some(ledger) = ledger {
16685            let peak = ordered.peak_heap_bytes.max(adapted.peak_heap_bytes);
16686            ledger
16687                .reserve_schedule_discovery(peak)
16688                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16689            ledger
16690                .release_scratch(peak)
16691                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16692        }
16693        // The planner's order is scratch now; a full recalc's is large.
16694        match self.thread_pool.as_deref() {
16695            Some(pool) if ordered.cells.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16696                pool.spawn(move || drop(ordered))
16697            }
16698            _ => drop(ordered),
16699        }
16700        Ok(adapted.schedule)
16701    }
16702
16703    /// Static-schedule cache eligibility. Legacy excludes range readers:
16704    /// their order comes from per-request range virtual deps. Under the
16705    /// authority a range read is a static edge of the relation, so only
16706    /// dynamic readers (whose hints are per request) are excluded, and the
16707    /// key adds the authority revision (design §8.4).
16708    fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
16709        {
16710            // A dynamic reader is planned from its observed reads, which the
16711            // key covers (rev.dyn); one without them needs a pre-probe, and
16712            // request-scoped replan hints are never cached.
16713            let host = self.graph.authority_host();
16714            !to_evaluate.is_empty()
16715                && !self.freshness_has_hints()
16716                && to_evaluate
16717                    .iter()
16718                    .all(|&v| !self.graph.is_dynamic(v) || host.observed(v).is_some())
16719        }
16720    }
16721
16722    fn schedule_cache_authority_revision(&self) -> (u64, u64) {
16723        {
16724            // rev.topology is `topology_epoch`; a symbol revision rebuilds the
16725            // store, so it is part of `revision`.
16726            let host = self.graph.authority_host();
16727            (host.revision(), host.rev_dyn())
16728        }
16729    }
16730
16731    fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
16732        VirtualDepTelemetry {
16733            fallback_mode_activations: self.virtual_dep_fallback_activations,
16734            ..VirtualDepTelemetry::default()
16735        }
16736    }
16737
16738    fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
16739        telemetry.candidate_vertices_total += meta.candidate_vertices;
16740        telemetry.vdeps_vertices_total += meta.vdeps_vertices;
16741        telemetry.vdeps_edges_total += meta.vdeps_edges;
16742        telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
16743        if meta.schedule_cache_eligible {
16744            if meta.schedule_cache_hit {
16745                telemetry.schedule_cache_hits += 1;
16746                telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
16747            } else {
16748                telemetry.schedule_cache_misses += 1;
16749            }
16750        }
16751        if meta.used_virtual_schedule {
16752            telemetry.schedule_virtual_passes += 1;
16753        } else {
16754            telemetry.schedule_static_passes += 1;
16755        }
16756    }
16757
16758    /// End-of-pass dirty bookkeeping; true when the loop must replan.
16759    /// Legacy: clear the pass, re-dirty readers whose pre-probe changed.
16760    /// Under the authority an armed pass keeps stale and unreached vertices
16761    /// dirty instead (design §8.2, `freshness.rs`).
16762    fn finish_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16763        self.finish_pass_dirty_scoped(to_evaluate, changed, true)
16764    }
16765
16766    /// [`Self::finish_pass_dirty`] for a targeted pass: only its candidates
16767    /// decide whether to replan.
16768    fn finish_target_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16769        self.finish_pass_dirty_scoped(to_evaluate, changed, false)
16770    }
16771
16772    fn finish_pass_dirty_scoped(
16773        &mut self,
16774        to_evaluate: &[VertexId],
16775        changed: &[VertexId],
16776        #[allow(unused_variables)] whole_workbook: bool,
16777    ) -> bool {
16778        self.freshness_finish_pass(to_evaluate, changed, whole_workbook)
16779    }
16780
16781    /// Start a pass over `schedule` (arms the freshness recorder).
16782    fn begin_pass(
16783        &mut self,
16784        #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16785    ) {
16786        self.freshness_begin_pass(schedule);
16787    }
16788
16789    /// FR4 layer barrier after unit `index`: true stops the pass.
16790    fn stop_after_unit(
16791        &mut self,
16792        #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16793        #[allow(unused_variables)] index: usize,
16794    ) -> bool {
16795        self.freshness_stop_after_unit(schedule, index)
16796    }
16797
16798    fn changed_virtual_dep_vertices(
16799        &mut self,
16800        to_evaluate: &[VertexId],
16801        old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16802    ) -> Vec<VertexId> {
16803        #[cfg(test)]
16804        if self.force_virtual_dep_changes_remaining_for_test > 0
16805            && let Some(vertex) = to_evaluate.first().copied()
16806        {
16807            self.force_virtual_dep_changes_remaining_for_test -= 1;
16808            return vec![vertex];
16809        }
16810        // An armed pass detects stale dynamic reads directly; the pre-probe
16811        // comparison is legacy's substitute for that (design §8.2).
16812        if self.freshness_armed() {
16813            return Vec::new();
16814        }
16815        if !to_evaluate
16816            .iter()
16817            .copied()
16818            .any(|v| self.graph.is_dynamic(v))
16819        {
16820            return Vec::new();
16821        }
16822
16823        let builder = VirtualDepBuilder::new(self);
16824        let (new_vdeps, _) = builder.build(to_evaluate);
16825
16826        let mut candidates = FxHashSet::default();
16827        candidates.extend(old_vdeps.keys().copied());
16828        candidates.extend(new_vdeps.keys().copied());
16829
16830        let mut changed = Vec::new();
16831        for v in candidates {
16832            if old_vdeps.get(&v) != new_vdeps.get(&v) {
16833                changed.push(v);
16834            }
16835        }
16836        changed
16837    }
16838
16839    /// Build a demand-driven subgraph for the given targets, including ephemeral edges for
16840    /// compressed ranges, and returning the set of dirty/volatile precedents and virtual deps.
16841    /// Demand candidates of `targets` and their dynamic plan hints: under
16842    /// `unified_authority` from the authority's relation (design §8.3),
16843    /// otherwise from the legacy graph.
16844    #[allow(clippy::type_complexity)]
16845    fn demand_subgraph(
16846        &self,
16847        targets: &[VertexId],
16848    ) -> Result<
16849        (
16850            Vec<VertexId>,
16851            rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16852        ),
16853        ExcelError,
16854    > {
16855        self.authority_demand_subgraph(targets)
16856    }
16857
16858    /// Design §8.3: traverse precedents from the targets over the
16859    /// authority's static relation (symbol nodes are ordinary pieces) plus
16860    /// the dynamic readers' virtual dependencies, and collect what legacy's
16861    /// demand walk collects: dirty or volatile formula cells and every name
16862    /// passed through. No legacy dependency structure is read.
16863    #[allow(clippy::type_complexity)]
16864    fn authority_demand_subgraph(
16865        &self,
16866        targets: &[VertexId],
16867    ) -> Result<
16868        (
16869            Vec<VertexId>,
16870            rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16871        ),
16872        ExcelError,
16873    > {
16874        use crate::engine::authority::geom::{Cell, Rect, SYMBOL_SHEET};
16875        use crate::engine::authority::store::TagFilter;
16876        use rustc_hash::{FxHashMap, FxHashSet};
16877        let store = self
16878            .graph
16879            .authority_plan_store()
16880            .map_err(Self::authority_excel_error)?;
16881        let ids = store.ids();
16882        let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
16883        let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default();
16884        let mut visited: FxHashSet<Cell> = FxHashSet::default();
16885        // (cell, authority id or NO_VID): the id lets the side array
16886        // translate the cell without a hash lookup.
16887        let mut stack: Vec<(Cell, u32)> = Vec::new();
16888        // Formula cells under `rect` on `sheet`: identity runs per column.
16889        let push_formulas = |stack: &mut Vec<(Cell, u32)>,
16890                             visited: &FxHashSet<Cell>,
16891                             sheet: u16,
16892                             rect: Rect| {
16893            for col in rect.c0..=rect.c1 {
16894                ids.visit_runs_in(sheet, col, rect.r0, rect.r1, &mut |h| {
16895                    let run = ids.run(h);
16896                    let r0 = run.row_start.max(rect.r0);
16897                    let r1 = (run.row_start + run.len - 1).min(rect.r1);
16898                    for row in r0..=r1 {
16899                        if !visited.contains(&(sheet, row, col)) {
16900                            stack.push(((sheet, row, col), run.first_id + (row - run.row_start)));
16901                        }
16902                    }
16903                });
16904            }
16905        };
16906        for &v in targets {
16907            if let Some(table) = self.graph.table_by_vertex(v) {
16908                // A table's demand is its range's, as legacy's table vertex
16909                // leads to the cells it covers (its symbol row has no
16910                // precedents).
16911                let (s, e) = (table.range.start, table.range.end);
16912                push_formulas(
16913                    &mut stack,
16914                    &visited,
16915                    s.sheet_id,
16916                    Rect::new(s.coord.row(), s.coord.col(), e.coord.row(), e.coord.col()),
16917                );
16918            } else if let Some(cell) = self.graph.authority_cell_of_vertex(v) {
16919                stack.push((cell, crate::engine::authority::identity::NO_VID));
16920            }
16921        }
16922        let mut hits = Vec::new();
16923        #[cfg(any(test, feature = "benchmark_internal"))]
16924        let (mut probe_vertices, mut probe_clean_formulas, mut probe_edges, mut probe_dynamic) =
16925            (0, 0, 0, 0);
16926        while let Some((cell, id)) = stack.pop() {
16927            if !visited.insert(cell) {
16928                continue;
16929            }
16930            let Some(v) = self.graph.authority_vertex_of_formula(id, cell) else {
16931                continue;
16932            };
16933            if !self.graph.vertex_exists(v) {
16934                continue;
16935            }
16936            #[cfg(any(test, feature = "benchmark_internal"))]
16937            {
16938                probe_vertices += 1;
16939            }
16940            match self.graph.get_vertex_kind(v) {
16941                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
16942                    if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
16943                        to_evaluate.insert(v);
16944                    } else {
16945                        #[cfg(any(test, feature = "benchmark_internal"))]
16946                        {
16947                            probe_clean_formulas += 1;
16948                        }
16949                    }
16950                }
16951                VertexKind::NamedScalar | VertexKind::NamedArray => {
16952                    to_evaluate.insert(v);
16953                }
16954                _ => {}
16955            }
16956            hits.clear();
16957            store.direct_precedents(cell, TagFilter::All, &mut hits);
16958            #[cfg(any(test, feature = "benchmark_internal"))]
16959            {
16960                probe_edges += hits.len();
16961            }
16962            for &(_, sheet, rect) in &hits {
16963                // DirtyExtents: a dirty spill anchor whose extent meets this
16964                // image is a demand precedent, ordered before `v` (§8.2).
16965                if sheet != SYMBOL_SHEET {
16966                    for anchor in self
16967                        .graph
16968                        .spill_anchors_in_region(sheet, rect.r0, rect.c0, rect.r1, rect.c1)
16969                    {
16970                        if anchor != v && self.graph.is_dirty(anchor) {
16971                            vdeps.entry(v).or_default().push(anchor);
16972                            if let Some(c) = self.graph.authority_cell_of_vertex(anchor) {
16973                                stack.push((c, crate::engine::authority::identity::NO_VID));
16974                            }
16975                        }
16976                    }
16977                }
16978                if sheet == SYMBOL_SHEET {
16979                    stack.extend((rect.r0..=rect.r1).map(|slot| {
16980                        (
16981                            (SYMBOL_SHEET, slot, 0),
16982                            crate::engine::authority::identity::NO_VID,
16983                        )
16984                    }));
16985                    continue;
16986                }
16987                push_formulas(&mut stack, &visited, sheet, rect);
16988            }
16989            if self.graph.is_dynamic(v) {
16990                #[cfg(any(test, feature = "benchmark_internal"))]
16991                {
16992                    probe_dynamic += 1;
16993                }
16994                // rdi_dyn: the observed reads are demand precedents.
16995                if let Some(reads) = self.graph.authority_host().observed(v) {
16996                    for &(sheet, r0, c0, r1, c1) in reads {
16997                        push_formulas(&mut stack, &visited, sheet, Rect::new(r0, c0, r1, c1));
16998                    }
16999                }
17000                let (vdeps_map, _) = VirtualDepBuilder::new(self).build(&[v]);
17001                // Pre-probe targets plus reads this request found dirty
17002                // (design §8.3: demand walks rdi ∪ hints).
17003                let hinted = self.freshness_hints(v).unwrap_or(&[]);
17004                if let Some(deps) = vdeps_map
17005                    .get(&v)
17006                    .map(|deps| deps.iter().chain(hinted))
17007                    .or(Some([].iter().chain(hinted)))
17008                {
17009                    for &u in deps {
17010                        vdeps.entry(v).or_default().push(u);
17011                        if let Some(c) = self.graph.authority_cell_of_vertex(u) {
17012                            stack.push((c, crate::engine::authority::identity::NO_VID));
17013                        }
17014                    }
17015                }
17016            }
17017        }
17018        let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
17019        result.sort_unstable();
17020        for deps in vdeps.values_mut() {
17021            deps.sort_unstable();
17022            deps.dedup();
17023        }
17024        #[cfg(any(test, feature = "benchmark_internal"))]
17025        {
17026            let mut probe = self.recalc_reuse_probe.lock().unwrap();
17027            probe.demand_builds += 1;
17028            probe.demand_vertices += probe_vertices;
17029            probe.demand_clean_formulas += probe_clean_formulas;
17030            probe.demand_explicit_edges += probe_edges;
17031            probe.demand_virtual_builder_calls += probe_dynamic;
17032        }
17033        Ok((result, vdeps))
17034    }
17035
17036    /// Helper: convert 1-based column index to Excel-style letters (1 -> A, 27 -> AA)
17037    fn col_to_letters(col: u32) -> String {
17038        col_letters_from_1based(col).expect("column index must be >= 1")
17039    }
17040
17041    /// Evaluate all dirty/volatile vertices with cancellation support
17042    pub fn evaluate_all_cancellable(
17043        &mut self,
17044        cancel: crate::engine::CancelToken,
17045    ) -> Result<EvalResult, ExcelError> {
17046        self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
17047            engine.observe_function_semantic_epoch()?;
17048            engine.active_cancel_flag = Some(cancel.clone());
17049            let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
17050            engine.active_cancel_flag = None;
17051            res
17052        })
17053    }
17054
17055    fn evaluate_all_cancellable_impl(
17056        &mut self,
17057        cancel_flag: &AtomicBool,
17058    ) -> Result<EvalResult, ExcelError> {
17059        let _source_cache = self.source_cache_session();
17060        self.validate_deterministic_mode()?;
17061        if self.config.defer_graph_building {
17062            self.build_graph_all()?;
17063        }
17064        if cancel_flag.load(Ordering::Relaxed) {
17065            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17066                .with_message("Evaluation cancelled before scheduling".to_string()));
17067        }
17068        self.require_unified_authority()?;
17069        self.begin_evaluation_request();
17070        self.reset_virtual_dep_telemetry_if_disabled();
17071        let start = crate::instant::FzInstant::now();
17072        let mut computed_vertices = 0;
17073        let mut cycle_errors = 0;
17074
17075        let mut replan_iterations = 0;
17076        const MAX_REPLAN: usize = 5;
17077        let mut telemetry = self
17078            .config
17079            .enable_virtual_dep_telemetry
17080            .then(|| self.start_virtual_dep_telemetry());
17081
17082        loop {
17083            if cancel_flag.load(Ordering::Relaxed) {
17084                if let Some(mut t) = telemetry {
17085                    t.bailout_reason = Some("cancelled");
17086                    t.replan_iterations = replan_iterations;
17087                    self.last_virtual_dep_telemetry = t;
17088                }
17089                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17090                    .with_message("Evaluation cancelled before scheduling".to_string()));
17091            }
17092
17093            let to_evaluate = self.graph.get_evaluation_vertices();
17094            if to_evaluate.is_empty() {
17095                if let Some(t) = telemetry.as_mut()
17096                    && t.bailout_reason.is_none()
17097                {
17098                    t.bailout_reason = Some("no_work");
17099                }
17100                break;
17101            }
17102
17103            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
17104            if let Some(t) = telemetry.as_mut() {
17105                Self::accumulate_schedule_meta(t, &meta);
17106            }
17107
17108            // Walk units in condensation order, checking cancellation between
17109            // units (formerly between cycles and between layers).
17110            self.begin_pass(&schedule);
17111            for (unit_index, &unit) in schedule.units.iter().enumerate() {
17112                match unit {
17113                    ScheduleUnit::Cycle(i) => {
17114                        // Check cancellation between cycles
17115                        if cancel_flag.load(Ordering::Relaxed) {
17116                            if let Some(mut t) = telemetry {
17117                                t.bailout_reason = Some("cancelled");
17118                                t.replan_iterations = replan_iterations;
17119                                self.last_virtual_dep_telemetry = t;
17120                            }
17121                            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
17122                                "Evaluation cancelled during cycle handling".to_string(),
17123                            ));
17124                        }
17125
17126                        if self.handle_cycle_unit(
17127                            schedule.unit_cycle(i),
17128                            None,
17129                            None,
17130                            Some(cancel_flag),
17131                        )? > 0
17132                        {
17133                            cycle_errors += 1;
17134                        }
17135                    }
17136                    ScheduleUnit::Layer(i) => {
17137                        let layer = schedule.unit_layer(i);
17138                        // Check cancellation between layers
17139                        if cancel_flag.load(Ordering::Relaxed) {
17140                            if let Some(mut t) = telemetry {
17141                                t.bailout_reason = Some("cancelled");
17142                                t.replan_iterations = replan_iterations;
17143                                self.last_virtual_dep_telemetry = t;
17144                            }
17145                            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17146                                .with_message("Evaluation cancelled between layers".to_string()));
17147                        }
17148
17149                        // Evaluate vertices in this layer (parallel or sequential)
17150                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
17151                            computed_vertices +=
17152                                self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
17153                        } else {
17154                            computed_vertices +=
17155                                self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
17156                        }
17157                    }
17158                }
17159                if self.stop_after_unit(&schedule, unit_index) {
17160                    break;
17161                }
17162            }
17163
17164            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
17165            if let Some(t) = telemetry.as_mut() {
17166                t.changed_vdeps_total += changed_vertices.len();
17167            }
17168            self.resource_checkpoint(0)?;
17169            if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
17170                if let Some(t) = telemetry.as_mut() {
17171                    t.bailout_reason = Some("converged");
17172                }
17173                break;
17174            }
17175            if replan_iterations >= MAX_REPLAN {
17176                if let Some(mut t) = telemetry.take() {
17177                    t.bailout_reason = Some("max_replan");
17178                    t.replan_iterations = replan_iterations;
17179                    self.last_virtual_dep_telemetry = t;
17180                }
17181                return Err(
17182                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
17183                );
17184            }
17185            replan_iterations += 1;
17186        }
17187
17188        if let Some(mut t) = telemetry {
17189            t.replan_iterations = replan_iterations;
17190            self.last_virtual_dep_telemetry = t;
17191        }
17192
17193        // Re-dirty volatile vertices for the next evaluation cycle
17194        self.redirty_for_next_recalc();
17195        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
17196
17197        Ok(EvalResult {
17198            computed_vertices,
17199            cycle_errors,
17200            elapsed: start.elapsed(),
17201        })
17202    }
17203
17204    /// Evaluate only the necessary precedents for specific target cells with cancellation support
17205    pub fn evaluate_until_cancellable(
17206        &mut self,
17207        targets: &[&str],
17208        cancel: crate::engine::CancelToken,
17209    ) -> Result<EvalResult, ExcelError> {
17210        self.observe_evaluation_resource_request(
17211            EvaluationRequestKind::TargetedCancellable,
17212            |engine| {
17213                engine.observe_function_semantic_epoch()?;
17214                engine.active_cancel_flag = Some(cancel.clone());
17215                let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
17216                engine.active_cancel_flag = None;
17217                res
17218            },
17219        )
17220    }
17221
17222    fn evaluate_until_cancellable_impl(
17223        &mut self,
17224        targets: &[&str],
17225        cancel_flag: &AtomicBool,
17226    ) -> Result<EvalResult, ExcelError> {
17227        if cancel_flag.load(Ordering::Relaxed) {
17228            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17229                .with_message("Evaluation cancelled before target preparation"));
17230        }
17231        let mut typed_targets = Vec::with_capacity(targets.len());
17232        for target in targets {
17233            let (sheet, row, col) = self.parse_a1_notation(target)?;
17234            self.graph.sheet_id_mut(&sheet);
17235            typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
17236        }
17237        self.evaluate_mixed_targets(&typed_targets, None)
17238    }
17239
17240    fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
17241        let mut quoted = false;
17242        let mut separator = None;
17243        let bytes = address.as_bytes();
17244        let mut index = 0usize;
17245        while index < bytes.len() {
17246            match bytes[index] {
17247                b'\'' => {
17248                    if quoted && bytes.get(index + 1) == Some(&b'\'') {
17249                        index = index.saturating_add(1);
17250                    } else {
17251                        quoted = !quoted;
17252                    }
17253                }
17254                b'!' if !quoted => separator = Some(index),
17255                _ => {}
17256            }
17257            index = index.saturating_add(1);
17258        }
17259        if quoted {
17260            return Err(ExcelError::new(ExcelErrorKind::Ref)
17261                .with_message(format!("Invalid quoted sheet reference `{address}`")));
17262        }
17263        let (sheet, cell_part) = match separator {
17264            Some(separator) => {
17265                let raw_sheet = &address[..separator];
17266                let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
17267                    raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
17268                } else {
17269                    raw_sheet.to_string()
17270                };
17271                (sheet, &address[separator + 1..])
17272            }
17273            None => (self.default_sheet_name().to_string(), address),
17274        };
17275
17276        let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
17277            ExcelError::new(ExcelErrorKind::Ref)
17278                .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
17279        })?;
17280
17281        Ok((sheet, row, col))
17282    }
17283
17284    /// Determine volatility using this engine's FunctionProvider, falling back to global registry.
17285    fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
17286        use formualizer_parse::parser::ASTNodeType;
17287        match &ast.node_type {
17288            ASTNodeType::Function { name, args, .. } => {
17289                if let Some(func) = self
17290                    .get_function("", name)
17291                    .or_else(|| crate::function_registry::get("", name))
17292                    && func.caps().contains(crate::function::FnCaps::VOLATILE)
17293                {
17294                    return true;
17295                }
17296                args.iter()
17297                    .any(|arg| self.is_ast_volatile_with_provider(arg))
17298            }
17299            ASTNodeType::BinaryOp { left, right, .. } => {
17300                self.is_ast_volatile_with_provider(left)
17301                    || self.is_ast_volatile_with_provider(right)
17302            }
17303            ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
17304            ASTNodeType::Array(rows) => rows.iter().any(|row| {
17305                row.iter()
17306                    .any(|cell| self.is_ast_volatile_with_provider(cell))
17307            }),
17308            _ => false,
17309        }
17310    }
17311
17312    /// Evaluate a layer sequentially
17313    fn evaluate_layer_sequential(
17314        &mut self,
17315        layer: &super::scheduler::Layer,
17316    ) -> Result<usize, ExcelError> {
17317        self.resource_checkpoint(layer.vertices.len() as u64)?;
17318        self.evaluate_layer_sequential_effects(layer)
17319    }
17320
17321    fn update_vertex_value_with_delta(
17322        &mut self,
17323        vertex_id: VertexId,
17324        new_value: LiteralValue,
17325        delta: &mut DeltaCollector,
17326    ) {
17327        if delta.mode != DeltaMode::Off
17328            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
17329        {
17330            let sheet_name = self.graph.sheet_name(cell.sheet_id);
17331            let old = self
17332                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
17333                .unwrap_or(LiteralValue::Empty);
17334            if old != new_value {
17335                delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17336            }
17337        }
17338        self.graph.update_vertex_value_ref(vertex_id, &new_value);
17339        self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
17340    }
17341
17342    fn evaluate_layer_sequential_with_delta(
17343        &mut self,
17344        layer: &super::scheduler::Layer,
17345        delta: &mut DeltaCollector,
17346    ) -> Result<usize, ExcelError> {
17347        self.resource_checkpoint(layer.vertices.len() as u64)?;
17348        self.evaluate_layer_sequential_with_delta_effects(layer, delta)
17349    }
17350
17351    /// Evaluate a layer sequentially with cancellation support
17352    fn evaluate_layer_sequential_cancellable(
17353        &mut self,
17354        layer: &super::scheduler::Layer,
17355        cancel_flag: &AtomicBool,
17356    ) -> Result<usize, ExcelError> {
17357        self.resource_checkpoint(layer.vertices.len() as u64)?;
17358        self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
17359    }
17360
17361    /// Evaluate a layer sequentially with more frequent cancellation checks for demand-driven evaluation
17362    fn evaluate_layer_sequential_cancellable_demand_driven(
17363        &mut self,
17364        layer: &super::scheduler::Layer,
17365        cancel_flag: &AtomicBool,
17366    ) -> Result<usize, ExcelError> {
17367        self.resource_checkpoint(layer.vertices.len() as u64)?;
17368        self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
17369    }
17370
17371    /// Evaluate a layer in parallel using the thread pool.
17372    ///
17373    /// Cost-adaptive: the layer starts sequentially in slices of doubling
17374    /// size (1, 2, 4, ... vertices, capped so a slice does not overshoot the
17375    /// probe) and hands the rest to the pool once the rest looks worth it
17376    /// (`PARALLEL_LAYER_WORTH` at the rate so far) or the probe
17377    /// (`PARALLEL_LAYER_PROBE`) is spent. A cheap layer never pays the pool's
17378    /// wake-up and join (most Enron layers are tens of µs of work); an
17379    /// expensive one goes parallel after a few vertices. Splitting a layer
17380    /// into consecutive sub-layers is a valid order: its vertices are
17381    /// independent.
17382    fn evaluate_layer_parallel(
17383        &mut self,
17384        layer: &super::scheduler::Layer,
17385    ) -> Result<usize, ExcelError> {
17386        if layer.sequential {
17387            return self.evaluate_layer_sequential(layer);
17388        }
17389        self.resource_checkpoint(layer.vertices.len() as u64)?;
17390        let len = layer.vertices.len();
17391        let buffered = buffer_layer_writes(layer);
17392        let (probe, worth) = (PARALLEL_LAYER_PROBE, PARALLEL_LAYER_WORTH);
17393        let start = crate::instant::FzInstant::now();
17394        let mut pos = 0usize;
17395        let mut step = 1usize;
17396        while pos < len {
17397            if pos > 0 {
17398                let elapsed = start.elapsed();
17399                // Rate so far (ns per vertex) and the rest at that rate.
17400                let per_vertex = elapsed.as_nanos() / pos as u128 + 1;
17401                let rest_estimate = per_vertex * (len - pos) as u128;
17402                if len - pos >= 2 && (elapsed >= probe || rest_estimate >= worth.as_nanos()) {
17403                    let rest = layer.sub_layer(pos, len);
17404                    // Expensive members (a SUMIF over a table) parallelize
17405                    // one per task; cheap ones keep runs of 8 together.
17406                    let min_chunk = if per_vertex >= EXPENSIVE_VERTEX_NS {
17407                        1
17408                    } else {
17409                        8
17410                    };
17411                    return Ok(pos + self.evaluate_layer_parallel_effects(&rest, min_chunk)?);
17412                }
17413                // Next slice: double, but no more than the rest of the probe
17414                // at the rate so far (a slice must not overshoot it).
17415                let fit = (probe.saturating_sub(elapsed).as_nanos() / per_vertex) as usize + 1;
17416                step = step.saturating_mul(2).min(fit);
17417            }
17418            let end = (pos + step).min(len);
17419            let slice = layer.sub_layer(pos, end);
17420            // A slice stops at the probe's end even if its members turn out
17421            // far more expensive than the rate so far predicted.
17422            pos += self.evaluate_layer_units_until(
17423                &slice,
17424                None,
17425                None,
17426                None,
17427                buffered,
17428                Some(start + probe),
17429            )?;
17430        }
17431        Ok(len)
17432    }
17433
17434    fn evaluate_layer_parallel_with_delta(
17435        &mut self,
17436        layer: &super::scheduler::Layer,
17437        delta: &mut DeltaCollector,
17438    ) -> Result<usize, ExcelError> {
17439        if layer.sequential {
17440            return self.evaluate_layer_sequential_with_delta(layer, delta);
17441        }
17442        self.resource_checkpoint(layer.vertices.len() as u64)?;
17443        self.evaluate_layer_parallel_with_delta_effects(layer, delta)
17444    }
17445
17446    /// Evaluate a layer in parallel with cancellation support
17447    fn evaluate_layer_parallel_cancellable(
17448        &mut self,
17449        layer: &super::scheduler::Layer,
17450        cancel_flag: &AtomicBool,
17451    ) -> Result<usize, ExcelError> {
17452        if layer.sequential {
17453            return self.evaluate_layer_sequential_cancellable(layer, cancel_flag);
17454        }
17455        self.resource_checkpoint(layer.vertices.len() as u64)?;
17456        self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
17457    }
17458
17459    /// Evaluate a single vertex without mutating the graph (for parallel evaluation)
17460    fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17461        // Check if vertex exists
17462        if !self.graph.vertex_exists(vertex_id) {
17463            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17464                .with_message(format!("Vertex not found: {vertex_id:?}")));
17465        }
17466
17467        // Get vertex kind and check if it needs evaluation
17468        let kind = self.graph.get_vertex_kind(vertex_id);
17469        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17470
17471        let view = match kind {
17472            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17473                if let Some(view) = self.graph.formula_view(vertex_id) {
17474                    view
17475                } else {
17476                    return Ok(LiteralValue::Number(0.0));
17477                }
17478            }
17479            VertexKind::Empty | VertexKind::Cell => {
17480                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17481                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17482                    let row = cell_ref.coord.row() + 1;
17483                    let col = cell_ref.coord.col() + 1;
17484                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17485                        return Ok(v);
17486                    }
17487                }
17488                return Ok(LiteralValue::Number(0.0));
17489            }
17490            VertexKind::NamedScalar => {
17491                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17492                    ExcelError::new(ExcelErrorKind::Name)
17493                        .with_message("Named range metadata missing".to_string())
17494                })?;
17495
17496                return match &named_range.definition {
17497                    NamedDefinition::Cell(cell_ref) => {
17498                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17499                        Ok(self
17500                            .read_cell_value(
17501                                sheet_name,
17502                                cell_ref.coord.row() + 1,
17503                                cell_ref.coord.col() + 1,
17504                            )
17505                            .unwrap_or(LiteralValue::Empty))
17506                    }
17507                    NamedDefinition::Literal(v) => Ok(v.clone()),
17508                    NamedDefinition::Formula { ast, .. } => {
17509                        let context_sheet = match named_range.scope {
17510                            NameScope::Sheet(id) => id,
17511                            NameScope::Workbook => sheet_id,
17512                        };
17513                        let sheet_name = self.graph.sheet_name(context_sheet);
17514                        let cell_ref = self
17515                            .graph
17516                            .get_cell_ref(vertex_id)
17517                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17518                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17519                            .with_resolved_functions(&self.resolved_functions)
17520                            .with_arena_cells(self);
17521                        interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
17522                    }
17523                    NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17524                        .with_message("Range-valued name evaluated as scalar".to_string())),
17525                };
17526            }
17527            VertexKind::NamedArray => {
17528                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17529                    ExcelError::new(ExcelErrorKind::Name)
17530                        .with_message("Named range metadata missing".to_string())
17531                })?;
17532
17533                return match &named_range.definition {
17534                    NamedDefinition::Range(range_ref) => {
17535                        if range_ref.start.sheet_id != range_ref.end.sheet_id {
17536                            return Err(ExcelError::new(ExcelErrorKind::Ref)
17537                                .with_message("Named range cannot span sheets".to_string()));
17538                        }
17539                        let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17540                        let sr0 = range_ref.start.coord.row();
17541                        let sc0 = range_ref.start.coord.col();
17542                        let er0 = range_ref.end.coord.row();
17543                        let ec0 = range_ref.end.coord.col();
17544                        if sr0 > er0 || sc0 > ec0 {
17545                            return Err(ExcelError::new(ExcelErrorKind::Ref)
17546                                .with_message("Invalid named range bounds".to_string()));
17547                        }
17548
17549                        let h = (er0 - sr0 + 1) as usize;
17550                        let w = (ec0 - sc0 + 1) as usize;
17551                        let cell_count = (h as u64).saturating_mul(w as u64);
17552                        if cell_count > self.config.spill.max_spill_cells as u64 {
17553                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17554                                "Named range too large to materialize as an array".to_string(),
17555                            ));
17556                        }
17557
17558                        // Read numeric serials, resolving the sheet once.
17559                        let sheet_id = range_ref.start.sheet_id;
17560                        let asheet = self.arrow_sheets.sheet(sheet_name);
17561                        let mut rows = Vec::with_capacity(h);
17562                        for r0 in sr0..=er0 {
17563                            let mut row = Vec::with_capacity(w);
17564                            for c0 in sc0..=ec0 {
17565                                row.push(
17566                                    self.read_cell_formatted_in(sheet_id, asheet, r0 + 1, c0 + 1)
17567                                        .0,
17568                                );
17569                            }
17570                            rows.push(row);
17571                        }
17572                        Ok(LiteralValue::Array(rows))
17573                    }
17574                    NamedDefinition::Cell(cell_ref) => {
17575                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17576                        let row = cell_ref.coord.row() + 1;
17577                        let col = cell_ref.coord.col() + 1;
17578                        let v = self
17579                            .read_cell_value(sheet_name, row, col)
17580                            .unwrap_or(LiteralValue::Empty);
17581                        Ok(LiteralValue::Array(vec![vec![v]]))
17582                    }
17583                    NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
17584                    NamedDefinition::Formula { ast, .. } => {
17585                        let context_sheet = match named_range.scope {
17586                            NameScope::Sheet(id) => id,
17587                            NameScope::Workbook => sheet_id,
17588                        };
17589                        let sheet_name = self.graph.sheet_name(context_sheet);
17590                        let cell_ref = self
17591                            .graph
17592                            .get_cell_ref(vertex_id)
17593                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17594                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17595                            .with_resolved_functions(&self.resolved_functions)
17596                            .with_arena_cells(self);
17597                        match interpreter.evaluate_ast(ast) {
17598                            Ok(cv) => {
17599                                let v = cv.into_literal();
17600                                match v {
17601                                    LiteralValue::Array(_) => Ok(v),
17602                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
17603                                }
17604                            }
17605                            Err(err) => Ok(LiteralValue::Error(err)),
17606                        }
17607                    }
17608                };
17609            }
17610            VertexKind::InfiniteRange
17611            | VertexKind::Range
17612            | VertexKind::External
17613            | VertexKind::Table => {
17614                // Not directly evaluatable here.
17615                return Ok(LiteralValue::Number(0.0));
17616            }
17617        };
17618
17619        // The interpreter uses a reference to the engine as the context
17620        let sheet_name = self.graph.sheet_name(sheet_id);
17621        let cell_ref = self
17622            .graph
17623            .get_cell_ref(vertex_id)
17624            .expect("cell ref for vertex");
17625        if let Some(result) =
17626            self.freshness_evaluate_recorded(vertex_id, sheet_name, cell_ref, view)
17627        {
17628            return result;
17629        }
17630        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17631            .with_resolved_functions(&self.resolved_functions)
17632            .with_arena_cells(self);
17633
17634        interpreter
17635            .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
17636            .map(|cv| {
17637                let format = cv.format_id();
17638                self.record_derived_format(vertex_id, format);
17639                self.materialize_formula_result(vertex_id, cv)
17640            })
17641            .or_else(|error| self.fit_formula_error(vertex_id, error))
17642    }
17643
17644    /// Get access to the shared thread pool for parallel evaluation
17645    pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
17646        self.thread_pool.as_ref()
17647    }
17648}
17649
17650#[derive(Default)]
17651struct RowBoundsCache {
17652    snapshot: u64,
17653    // key: (sheet_id, col_idx)
17654    map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
17655}
17656
17657impl RowBoundsCache {
17658    fn new(snapshot: u64) -> Self {
17659        Self {
17660            snapshot,
17661            map: Default::default(),
17662        }
17663    }
17664    fn get_row_bounds(
17665        &self,
17666        sheet_id: SheetId,
17667        col_idx: usize,
17668        snapshot: u64,
17669    ) -> Option<(Option<u32>, Option<u32>)> {
17670        if self.snapshot != snapshot {
17671            return None;
17672        }
17673        self.map.get(&(sheet_id as u32, col_idx)).copied()
17674    }
17675    fn put_row_bounds(
17676        &mut self,
17677        sheet_id: SheetId,
17678        col_idx: usize,
17679        snapshot: u64,
17680        bounds: (Option<u32>, Option<u32>),
17681    ) {
17682        if self.snapshot != snapshot {
17683            self.snapshot = snapshot;
17684            self.map.clear();
17685        }
17686        self.map.insert((sheet_id as u32, col_idx), bounds);
17687    }
17688}
17689
17690struct UsedAxisBoundsCache {
17691    snapshot: u64,
17692    row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17693    col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17694    #[cfg(test)]
17695    row_hits: std::sync::atomic::AtomicUsize,
17696    #[cfg(test)]
17697    row_misses: std::sync::atomic::AtomicUsize,
17698    #[cfg(test)]
17699    col_hits: std::sync::atomic::AtomicUsize,
17700    #[cfg(test)]
17701    col_misses: std::sync::atomic::AtomicUsize,
17702}
17703
17704impl UsedAxisBoundsCache {
17705    fn new(snapshot: u64) -> Self {
17706        Self {
17707            snapshot,
17708            row_bounds_by_col_span: Default::default(),
17709            col_bounds_by_row_span: Default::default(),
17710            #[cfg(test)]
17711            row_hits: std::sync::atomic::AtomicUsize::new(0),
17712            #[cfg(test)]
17713            row_misses: std::sync::atomic::AtomicUsize::new(0),
17714            #[cfg(test)]
17715            col_hits: std::sync::atomic::AtomicUsize::new(0),
17716            #[cfg(test)]
17717            col_misses: std::sync::atomic::AtomicUsize::new(0),
17718        }
17719    }
17720
17721    fn reset_for_snapshot(&mut self, snapshot: u64) {
17722        if self.snapshot != snapshot {
17723            self.snapshot = snapshot;
17724            self.row_bounds_by_col_span.clear();
17725            self.col_bounds_by_row_span.clear();
17726        }
17727    }
17728
17729    fn get_row_bounds(
17730        &self,
17731        sheet_id: SheetId,
17732        start_col: u32,
17733        end_col: u32,
17734        snapshot: u64,
17735    ) -> Option<Option<(u32, u32)>> {
17736        if self.snapshot != snapshot {
17737            return None;
17738        }
17739        let cached = self
17740            .row_bounds_by_col_span
17741            .get(&(sheet_id, start_col, end_col))
17742            .copied();
17743        #[cfg(test)]
17744        if cached.is_some() {
17745            self.row_hits.fetch_add(1, Ordering::Relaxed);
17746        }
17747        cached
17748    }
17749
17750    fn put_row_bounds(
17751        &mut self,
17752        sheet_id: SheetId,
17753        start_col: u32,
17754        end_col: u32,
17755        snapshot: u64,
17756        bounds: Option<(u32, u32)>,
17757    ) {
17758        self.reset_for_snapshot(snapshot);
17759        self.row_bounds_by_col_span
17760            .insert((sheet_id, start_col, end_col), bounds);
17761        #[cfg(test)]
17762        self.row_misses.fetch_add(1, Ordering::Relaxed);
17763    }
17764
17765    fn get_col_bounds(
17766        &self,
17767        sheet_id: SheetId,
17768        start_row: u32,
17769        end_row: u32,
17770        snapshot: u64,
17771    ) -> Option<Option<(u32, u32)>> {
17772        if self.snapshot != snapshot {
17773            return None;
17774        }
17775        let cached = self
17776            .col_bounds_by_row_span
17777            .get(&(sheet_id, start_row, end_row))
17778            .copied();
17779        #[cfg(test)]
17780        if cached.is_some() {
17781            self.col_hits.fetch_add(1, Ordering::Relaxed);
17782        }
17783        cached
17784    }
17785
17786    fn put_col_bounds(
17787        &mut self,
17788        sheet_id: SheetId,
17789        start_row: u32,
17790        end_row: u32,
17791        snapshot: u64,
17792        bounds: Option<(u32, u32)>,
17793    ) {
17794        self.reset_for_snapshot(snapshot);
17795        self.col_bounds_by_row_span
17796            .insert((sheet_id, start_row, end_row), bounds);
17797        #[cfg(test)]
17798        self.col_misses.fetch_add(1, Ordering::Relaxed);
17799    }
17800}
17801
17802// Phase 2 shim: in-process spill manager delegating to current graph methods.
17803#[derive(Default)]
17804pub struct ShimSpillManager {
17805    region_locks: RegionLockManager,
17806    pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
17807}
17808
17809impl ShimSpillManager {
17810    pub(crate) fn reserve(
17811        &mut self,
17812        owner: VertexId,
17813        anchor_cell: CellRef,
17814        shape: SpillShape,
17815        _meta: SpillMeta,
17816    ) -> Result<(), ExcelError> {
17817        // Derive region from anchor + shape; enforce in-flight exclusivity only.
17818        let region = crate::engine::spill::Region {
17819            sheet_id: anchor_cell.sheet_id as u32,
17820            row_start: anchor_cell.coord.row(),
17821            row_end: anchor_cell
17822                .coord
17823                .row()
17824                .saturating_add(shape.rows)
17825                .saturating_sub(1),
17826            col_start: anchor_cell.coord.col(),
17827            col_end: anchor_cell
17828                .coord
17829                .col()
17830                .saturating_add(shape.cols)
17831                .saturating_sub(1),
17832        };
17833        match self.region_locks.reserve(region, owner) {
17834            Ok(id) => {
17835                if id != 0 {
17836                    self.active_locks.insert(owner, id);
17837                }
17838                Ok(())
17839            }
17840            Err(e) => Err(e),
17841        }
17842    }
17843
17844    /// Release any in-flight region reservation still held for `owner`.
17845    ///
17846    /// Reservations are normally released on commit/rollback, but if an anchor is
17847    /// abandoned without committing (e.g. cycle detection stamps it with #CIRC), a
17848    /// stale reservation could remain. This is a no-op when nothing is held.
17849    pub(crate) fn release_owner(&mut self, owner: VertexId) {
17850        if let Some(id) = self.active_locks.remove(&owner) {
17851            self.region_locks.release(id);
17852        }
17853    }
17854
17855    pub(crate) fn commit_array_with_value_probe<F>(
17856        &mut self,
17857        graph: &mut DependencyGraph,
17858        anchor_vertex: VertexId,
17859        targets: &[CellRef],
17860        rows: Vec<Vec<LiteralValue>>,
17861        mut value_probe: F,
17862    ) -> Result<(), ExcelError>
17863    where
17864        F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
17865    {
17866        use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
17867
17868        // Re-run plan on concrete targets before committing to respect blockers.
17869        // This plan checks formula/spill ownership in the graph, but when the graph value cache
17870        // is disabled (Arrow-canonical mode), it cannot see non-empty value blockers.
17871        let plan_res = graph.plan_spill_region(anchor_vertex, targets);
17872        if let Err(e) = plan_res {
17873            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17874                self.region_locks.release(id);
17875            }
17876            return Err(e);
17877        }
17878
17879        if !graph.value_cache_enabled() {
17880            // Compute expected spill shape from the target rectangle for diagnostics.
17881            let (expected_rows, expected_cols) = if targets.is_empty() {
17882                (0u32, 0u32)
17883            } else {
17884                let mut min_r = u32::MAX;
17885                let mut max_r = 0u32;
17886                let mut min_c = u32::MAX;
17887                let mut max_c = 0u32;
17888                for cell in targets {
17889                    let r = cell.coord.row();
17890                    let c = cell.coord.col();
17891                    min_r = min_r.min(r);
17892                    max_r = max_r.max(r);
17893                    min_c = min_c.min(c);
17894                    max_c = max_c.max(c);
17895                }
17896                (
17897                    max_r.saturating_sub(min_r).saturating_add(1),
17898                    max_c.saturating_sub(min_c).saturating_add(1),
17899                )
17900            };
17901
17902            let anchor_cell = graph
17903                .get_cell_ref(anchor_vertex)
17904                .expect("anchor cell ref for spill commit");
17905
17906            for cell in targets {
17907                // Never treat the anchor as a blocker.
17908                if *cell == anchor_cell {
17909                    continue;
17910                }
17911                // Skip cells already known to be owned by a spill; plan() handled spill conflicts.
17912                if graph.spill_registry_anchor_for_cell(*cell).is_some() {
17913                    continue;
17914                }
17915                // Skip formula vertices in the target region; plan() handled them (or allowed).
17916                if let Some(vid) = graph.get_vertex_id_for_address(cell)
17917                    && vid != anchor_vertex
17918                {
17919                    match graph.get_vertex_kind(vid) {
17920                        crate::engine::vertex::VertexKind::FormulaScalar
17921                        | crate::engine::vertex::VertexKind::FormulaArray => {
17922                            // plan() already approved allowed overwrites.
17923                            continue;
17924                        }
17925                        _ => {}
17926                    }
17927                }
17928
17929                if let Some(v) = value_probe(graph, cell)
17930                    && !matches!(v, LiteralValue::Empty)
17931                {
17932                    if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17933                        self.region_locks.release(id);
17934                    }
17935                    return Err(ExcelError::new(ExcelErrorKind::Spill)
17936                        .with_message("BlockedByValue")
17937                        .with_extra(ExcelErrorExtra::Spill {
17938                            expected_rows,
17939                            expected_cols,
17940                        }));
17941                }
17942            }
17943        }
17944
17945        let commit_res = graph.commit_spill_region_atomic_with_fault(
17946            anchor_vertex,
17947            targets.to_vec(),
17948            rows,
17949            None,
17950        );
17951        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17952            self.region_locks.release(id);
17953        }
17954        commit_res.map(|_| ())
17955    }
17956
17957    /// Commit a spill and mirror all written cells into Arrow overlay via the owning engine.
17958    pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
17959        &mut self,
17960        engine: &mut Engine<R>,
17961        anchor_vertex: VertexId,
17962        targets: &[CellRef],
17963        rows: Vec<Vec<LiteralValue>>,
17964    ) -> Result<(), ExcelError> {
17965        if let Err(error) = engine.guard_pending_spill_commit(anchor_vertex, targets) {
17966            self.release_owner(anchor_vertex);
17967            return Err(error);
17968        }
17969        // Re-run plan on concrete targets before committing to respect blockers.
17970        let plan_res = engine.graph.plan_spill_region(anchor_vertex, targets);
17971        if let Err(e) = plan_res {
17972            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17973                self.region_locks.release(id);
17974            }
17975            return Err(e);
17976        }
17977
17978        let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
17979            anchor_vertex,
17980            targets.to_vec(),
17981            rows.clone(),
17982            None,
17983        );
17984        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17985            self.region_locks.release(id);
17986        }
17987        commit_res.map(|_| ())?;
17988        engine
17989            .blocked_pending_spills
17990            .retain(|entry| entry.0 != anchor_vertex);
17991        engine.spill_blocker_waiters.forget(anchor_vertex);
17992
17993        // Mirror into Arrow overlay when enabled
17994        if engine.config.arrow_storage_enabled
17995            && engine.config.delta_overlay_enabled
17996            && engine.config.write_formula_overlay_enabled
17997        {
17998            // Expect targets to be a contiguous rectangle row-major starting at some anchor
17999            for (idx, cell) in targets.iter().enumerate() {
18000                let (r_off, c_off) = {
18001                    if rows.is_empty() || rows[0].is_empty() {
18002                        (0usize, 0usize)
18003                    } else {
18004                        let width = rows[0].len();
18005                        (idx / width, idx % width)
18006                    }
18007                };
18008                let v = rows
18009                    .get(r_off)
18010                    .and_then(|r| r.get(c_off))
18011                    .cloned()
18012                    .unwrap_or(LiteralValue::Empty);
18013                let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
18014                engine.mirror_value_to_computed_overlay(
18015                    &sheet_name,
18016                    cell.coord.row() + 1,
18017                    cell.coord.col() + 1,
18018                    &v,
18019                );
18020            }
18021        }
18022        Ok(())
18023    }
18024}
18025
18026impl<R> Engine<R>
18027where
18028    R: EvaluationContext,
18029{
18030    fn resolve_shared_ref(
18031        &self,
18032        reference: &ReferenceType,
18033        current_sheet: &str,
18034    ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
18035        use formualizer_common::{
18036            SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
18037            SheetRef as SharedRef,
18038        };
18039
18040        // Preserve anchor flags from the parsed reference when possible.
18041        let sr = match reference {
18042            ReferenceType::Cell {
18043                sheet,
18044                row,
18045                col,
18046                row_abs,
18047                col_abs,
18048            } => {
18049                let row0 = row
18050                    .checked_sub(1)
18051                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18052                let col0 = col
18053                    .checked_sub(1)
18054                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18055                let sheet_loc = match sheet.as_deref() {
18056                    Some(name) => SheetLocator::from_name(name),
18057                    None => SheetLocator::Current,
18058                };
18059                let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
18060                SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
18061            }
18062            ReferenceType::Range {
18063                sheet,
18064                start_row,
18065                start_col,
18066                end_row,
18067                end_col,
18068                start_row_abs,
18069                start_col_abs,
18070                end_row_abs,
18071                end_col_abs,
18072            } => {
18073                let sheet_loc = match sheet.as_deref() {
18074                    Some(name) => SheetLocator::from_name(name),
18075                    None => SheetLocator::Current,
18076                };
18077                let sr = start_row
18078                    .map(|r| {
18079                        r.checked_sub(1)
18080                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18081                    })
18082                    .transpose()?;
18083                let sc = start_col
18084                    .map(|c| {
18085                        c.checked_sub(1)
18086                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18087                    })
18088                    .transpose()?;
18089                let er = end_row
18090                    .map(|r| {
18091                        r.checked_sub(1)
18092                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18093                    })
18094                    .transpose()?;
18095                let ec = end_col
18096                    .map(|c| {
18097                        c.checked_sub(1)
18098                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18099                    })
18100                    .transpose()?;
18101                let range = SharedRangeRef::from_parts(
18102                    sheet_loc,
18103                    sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
18104                    sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
18105                    er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
18106                    ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
18107                )
18108                .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
18109                SharedRef::Range(range)
18110            }
18111            _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18112        };
18113
18114        let current_id = self
18115            .graph
18116            .sheet_id(current_sheet)
18117            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18118
18119        let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
18120            match loc {
18121                SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
18122                SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
18123                SheetLocator::Name(name) => {
18124                    let n = name.as_ref();
18125                    self.graph
18126                        .sheet_id(n)
18127                        .map(SheetLocator::Id)
18128                        .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18129                }
18130            }
18131        };
18132
18133        match sr {
18134            SharedRef::Cell(cell) => {
18135                let owned = cell.into_owned();
18136                let sheet = resolve_loc(owned.sheet)?;
18137                Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
18138            }
18139            SharedRef::Range(range) => {
18140                let owned = range.into_owned();
18141                let sheet = resolve_loc(owned.sheet)?;
18142                Ok(SharedRef::Range(SharedRangeRef {
18143                    sheet,
18144                    start_row: owned.start_row,
18145                    start_col: owned.start_col,
18146                    end_row: owned.end_row,
18147                    end_col: owned.end_col,
18148                }))
18149            }
18150        }
18151    }
18152}
18153
18154// Implement the resolver traits for the Engine.
18155// This allows the interpreter to resolve references by querying the engine's graph.
18156impl<R> crate::traits::ReferenceResolver for Engine<R>
18157where
18158    R: EvaluationContext,
18159{
18160    fn resolve_cell_reference(
18161        &self,
18162        sheet: Option<&str>,
18163        row: u32,
18164        col: u32,
18165    ) -> Result<LiteralValue, ExcelError> {
18166        // This context-free trait method has no knowledge of the formula's
18167        // current sheet, so an unqualified (`None`) reference cannot be resolved
18168        // here. Previously this fell back to `default_sheet_name()`, which leaked
18169        // the reference onto an unrelated sheet (issue #110). Interpreter paths
18170        // already qualify references with the current sheet before reaching this
18171        // method (see `Interpreter::implicit_intersection_from_reference`), and
18172        // the sheet-aware scalar path goes through `resolve_cell_reference_value`
18173        // with an explicit `current_sheet`. Returning #REF! for an unqualified
18174        // reference here surfaces the missing context instead of silently
18175        // returning data from the wrong sheet.
18176        let Some(sheet_name) = sheet else {
18177            return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18178                "Unqualified cell reference resolved without sheet context".to_string(),
18179            ));
18180        };
18181        // Prefer engine's unified accessor which consults Arrow store for base values
18182        // and falls back to graph for formulas and stored values.
18183        if let Some(v) = self.read_cell_value(sheet_name, row, col) {
18184            Ok(v)
18185        } else {
18186            // Excel semantics: empty cell coerces to 0 in numeric contexts
18187            Ok(LiteralValue::Number(0.0))
18188        }
18189    }
18190}
18191
18192impl<R> crate::traits::RangeResolver for Engine<R>
18193where
18194    R: EvaluationContext,
18195{
18196    fn resolve_range_reference(
18197        &self,
18198        sheet: Option<&str>,
18199        sr: Option<u32>,
18200        sc: Option<u32>,
18201        er: Option<u32>,
18202        ec: Option<u32>,
18203    ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
18204        // For now, delegate range resolution to the external resolver.
18205        // A future optimization could be to handle this within the graph.
18206        self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
18207    }
18208}
18209
18210impl<R> crate::traits::NamedRangeResolver for Engine<R>
18211where
18212    R: EvaluationContext,
18213{
18214    fn resolve_named_range_reference(
18215        &self,
18216        name: &str,
18217    ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
18218        self.resolver.resolve_named_range_reference(name)
18219    }
18220}
18221
18222impl<R> crate::traits::TableResolver for Engine<R>
18223where
18224    R: EvaluationContext,
18225{
18226    fn resolve_table_reference(
18227        &self,
18228        tref: &formualizer_parse::parser::TableReference,
18229    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
18230        self.resolver.resolve_table_reference(tref)
18231    }
18232}
18233
18234impl<R> crate::traits::SourceResolver for Engine<R>
18235where
18236    R: EvaluationContext,
18237{
18238    fn source_scalar_version(&self, name: &str) -> Option<u64> {
18239        self.resolver.source_scalar_version(name)
18240    }
18241
18242    fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
18243        self.resolver.resolve_source_scalar(name)
18244    }
18245
18246    fn source_table_version(&self, name: &str) -> Option<u64> {
18247        self.resolver.source_table_version(name)
18248    }
18249
18250    fn resolve_source_table(
18251        &self,
18252        name: &str,
18253    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
18254        self.resolver.resolve_source_table(name)
18255    }
18256}
18257
18258// The Engine is a Resolver because it implements the constituent traits.
18259impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
18260
18261// The Engine provides functions by delegating to its internal resolver.
18262impl<R: EvaluationContext> crate::interpreter::ArenaCellResolver for Engine<R> {
18263    fn resolve_arena_range(
18264        &self,
18265        reference: &crate::engine::arena::CompactRefType,
18266        current: SheetId,
18267        row_delta: i64,
18268        col_delta: i64,
18269    ) -> Result<RangeView<'_>, ExcelError> {
18270        use crate::engine::arena::{CompactRefType, SheetKey};
18271        let CompactRefType::Range {
18272            sheet,
18273            start_row,
18274            start_col,
18275            end_row,
18276            end_col,
18277            start_row_abs,
18278            start_col_abs,
18279            end_row_abs,
18280            end_col_abs,
18281        } = reference
18282        else {
18283            return Err(ExcelError::new(ExcelErrorKind::Ref));
18284        };
18285        let sheet_id = match sheet {
18286            None => current,
18287            Some(SheetKey::Id(id)) => *id,
18288            Some(SheetKey::Name(id)) => self
18289                .graph
18290                .sheet_id(self.graph.data_store().resolve_ast_string(*id))
18291                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?,
18292        };
18293        if self.graph.sheet_reg().name(sheet_id).is_empty() {
18294            return Err(ExcelError::new(ExcelErrorKind::Ref));
18295        }
18296        let axis = |value, sentinel, delta, absolute| {
18297            if value == sentinel {
18298                Ok(None)
18299            } else {
18300                crate::interpreter::shift_axis_for_offset(value, delta, absolute).map(Some)
18301            }
18302        };
18303        let sr = axis(*start_row, 0, row_delta, *start_row_abs)?;
18304        let sc = axis(*start_col, 0, col_delta, *start_col_abs)?;
18305        let er = axis(*end_row, u32::MAX, row_delta, *end_row_abs)?;
18306        let ec = axis(*end_col, u32::MAX, col_delta, *end_col_abs)?;
18307        if sr.zip(er).is_some_and(|(s, e)| s > e) || sc.zip(ec).is_some_and(|(s, e)| s > e) {
18308            return Err(ExcelError::new(ExcelErrorKind::Ref));
18309        }
18310        self.resolve_range_coordinates(sheet_id, sr, sc, er, ec)
18311    }
18312
18313    fn resolve_arena_cell(
18314        &self,
18315        sheet: Option<crate::engine::arena::SheetKey>,
18316        current: crate::SheetId,
18317        row: u32,
18318        col: u32,
18319    ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
18320        use crate::engine::arena::SheetKey;
18321        let (sheet_id, named_arrow) = match sheet {
18322            None => (current, None),
18323            Some(SheetKey::Id(id)) => {
18324                if self.graph.sheet_reg().name(id).is_empty() {
18325                    return Err(ExcelError::new(ExcelErrorKind::Ref));
18326                }
18327                (id, None)
18328            }
18329            Some(SheetKey::Name(id)) => {
18330                let (sheet_id, arrow) = self
18331                    .resolved_sheet_names
18332                    .get(&id)
18333                    .copied()
18334                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18335                (sheet_id, Some(arrow))
18336            }
18337        };
18338        let arrow = named_arrow.unwrap_or_else(|| {
18339            self.resolved_arrow_sheets
18340                .get(usize::from(sheet_id))
18341                .copied()
18342                .flatten()
18343        });
18344        let asheet = arrow
18345            .and_then(|index| self.arrow_sheets.sheets.get(index))
18346            .or_else(|| {
18347                // Graph-only sheets can acquire Arrow storage when an earlier layer
18348                // commits its first formula result. The pass snapshot cannot hold an
18349                // index for that storage yet; retain the original lookup in this case.
18350                let name = match sheet {
18351                    Some(SheetKey::Name(id)) => self.graph.data_store().resolve_ast_string(id),
18352                    _ => self.graph.sheet_name(sheet_id),
18353                };
18354                self.arrow_sheets.sheet(name)
18355            });
18356        Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
18357    }
18358}
18359
18360impl<R> crate::traits::FunctionProvider for Engine<R>
18361where
18362    R: EvaluationContext,
18363{
18364    fn planning_semantic_revision(&self) -> Option<u64> {
18365        self.resolver.planning_semantic_revision()
18366    }
18367
18368    fn get_function(
18369        &self,
18370        prefix: &str,
18371        name: &str,
18372    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
18373        self.resolver.get_function(prefix, name)
18374    }
18375
18376    fn get_function_for_planning(
18377        &self,
18378        prefix: &str,
18379        name: &str,
18380    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
18381        self.resolver.get_function_for_planning(prefix, name)
18382    }
18383}
18384
18385impl<R> Engine<R>
18386where
18387    R: EvaluationContext,
18388{
18389    fn resolve_range_coordinates(
18390        &self,
18391        sheet_id: SheetId,
18392        sr: Option<u32>,
18393        sc: Option<u32>,
18394        er: Option<u32>,
18395        ec: Option<u32>,
18396    ) -> Result<RangeView<'_>, ExcelError> {
18397        let sheet_name = self.graph.sheet_name(sheet_id);
18398        let extent = resolve_used_extent_with_fallback(
18399            OpenRangeBounds {
18400                start_row: sr,
18401                start_column: sc,
18402                end_row: er,
18403                end_column: ec,
18404            },
18405            ExtentPolicy::EvaluationCompat {
18406                fallback_row: None,
18407                fallback_column: None,
18408            },
18409            || {
18410                self.sheet_bounds(sheet_name)
18411                    .map(|_| self.config.max_open_ended_rows)
18412            },
18413            || {
18414                self.sheet_bounds(sheet_name)
18415                    .map(|_| self.config.max_open_ended_cols)
18416            },
18417            |first, last| self.used_rows_for_columns(sheet_name, first, last),
18418            |first, last| self.used_cols_for_rows(sheet_name, first, last),
18419        );
18420        let (sr, sc, er, ec) = extent
18421            .map(|e| (e.start_row, e.start_column, e.end_row, e.end_column))
18422            .unwrap_or((1, 1, 0, 0));
18423        if self.force_materialize_range_views {
18424            if er < sr || ec < sc {
18425                return Ok(RangeView::from_owned_rows(
18426                    Vec::new(),
18427                    self.config.date_system,
18428                ));
18429            }
18430            let h = (er - sr + 1) as u64;
18431            let w = (ec - sc + 1) as u64;
18432            if h.saturating_mul(w) <= self.config.spill.max_spill_cells as u64 {
18433                let mut rows = Vec::with_capacity(h as usize);
18434                for r in sr..=er {
18435                    let mut row = Vec::with_capacity(w as usize);
18436                    for c in sc..=ec {
18437                        row.push(
18438                            self.read_cell_value(sheet_name, r, c)
18439                                .unwrap_or(LiteralValue::Empty),
18440                        );
18441                    }
18442                    rows.push(row);
18443                }
18444                return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
18445            }
18446        }
18447        let Some(asheet) = self.arrow_sheet_by_id(sheet_id) else {
18448            return Ok(RangeView::from_owned_rows(
18449                Vec::new(),
18450                self.config.date_system,
18451            ));
18452        };
18453        Ok(if er < sr || ec < sc {
18454            asheet.range_view(1, 1, 0, 0)
18455        } else {
18456            asheet.range_view(
18457                sr.saturating_sub(1) as usize,
18458                sc.saturating_sub(1) as usize,
18459                er.saturating_sub(1) as usize,
18460                ec.saturating_sub(1) as usize,
18461            )
18462        })
18463    }
18464
18465    /// Semantic used coordinates exclude graph-only dependency placeholders.
18466    ///
18467    /// Non-empty base/overlay/computed cells come from Arrow storage, while
18468    /// scalar and array formulas come from graph formula kinds even before
18469    /// their results are materialized. The legacy graph fallback is omitted:
18470    /// `load_packed_to_vertex` entries are either represented by those sources
18471    /// or are `Empty` dependency placeholders, not a third value authority.
18472    pub(crate) fn semantic_used_rows_for_columns(
18473        &self,
18474        sheet: &str,
18475        start_col: u32,
18476        end_col: u32,
18477    ) -> Option<(u32, u32)> {
18478        let arrow_bounds = self
18479            .sheet_store()
18480            .sheet(sheet)
18481            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18482        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18483        Self::union_used_bounds(arrow_bounds, formula_bounds)
18484    }
18485
18486    pub(crate) fn semantic_used_cols_for_rows(
18487        &self,
18488        sheet: &str,
18489        start_row: u32,
18490        end_row: u32,
18491    ) -> Option<(u32, u32)> {
18492        let arrow_bounds = self
18493            .sheet_store()
18494            .sheet(sheet)
18495            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18496        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18497        Self::union_used_bounds(arrow_bounds, formula_bounds)
18498    }
18499}
18500
18501// Override EvaluationContext to provide thread pool access
18502impl<R> crate::traits::EvaluationContext for Engine<R>
18503where
18504    R: EvaluationContext,
18505{
18506    fn clock(&self) -> &dyn crate::timezone::ClockProvider {
18507        &self.clock
18508    }
18509
18510    fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
18511        self.thread_pool.as_ref()
18512    }
18513
18514    fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
18515        self.active_cancel_flag.clone()
18516    }
18517
18518    fn chunk_hint(&self) -> Option<usize> {
18519        // Use a simple heuristic from configuration (stripe width * height) as a default hint.
18520        let hint =
18521            (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
18522        Some(hint.clamp(1024, 1 << 20)) // clamp between 1K and ~1M
18523    }
18524
18525    fn materialized_array_cells(&self) -> u64 {
18526        self.config
18527            .evaluation_budgets
18528            .scratch
18529            .materialized_array_cells
18530            .unwrap_or(crate::engine::resource_ledger::DEFAULT_MATERIALIZED_ARRAY_CELLS)
18531    }
18532
18533    fn volatile_level(&self) -> crate::traits::VolatileLevel {
18534        self.config.volatile_level
18535    }
18536
18537    fn workbook_seed(&self) -> u64 {
18538        self.config.workbook_seed
18539    }
18540
18541    fn recalc_epoch(&self) -> u64 {
18542        self.recalc_epoch
18543    }
18544
18545    fn workbook_sheet_count(&self) -> Option<usize> {
18546        Some(self.graph.sheet_reg().active_len())
18547    }
18548
18549    fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
18550        self.graph.sheet_reg().active_position(sheet)
18551    }
18552
18553    fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
18554        self.sheet_index_by_name(current_sheet)
18555    }
18556
18557    fn inspect_reference(
18558        &self,
18559        reference: &ReferenceType,
18560        current_sheet: &str,
18561    ) -> Result<Option<ReferenceInfo>, ExcelError> {
18562        let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
18563            let sheet_id = self
18564                .graph
18565                .sheet_id(sheet_name)
18566                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18567            let sheet_index = self
18568                .graph
18569                .sheet_reg()
18570                .active_position_by_id(sheet_id)
18571                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18572            Ok((sheet_id, sheet_index))
18573        };
18574
18575        let cell_info =
18576            |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
18577                let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18578                let row0 = row
18579                    .checked_sub(1)
18580                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18581                let col0 = col
18582                    .checked_sub(1)
18583                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18584                Ok(ReferenceInfo {
18585                    first_sheet_index: Some(sheet_index),
18586                    sheet_count: Some(1),
18587                    first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18588                })
18589            };
18590
18591        let range_info = |sheet_name: &str,
18592                          start_row: Option<u32>,
18593                          start_col: Option<u32>|
18594         -> Result<ReferenceInfo, ExcelError> {
18595            let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18596            let row = start_row.unwrap_or(1);
18597            let col = start_col.unwrap_or(1);
18598            let row0 = row
18599                .checked_sub(1)
18600                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18601            let col0 = col
18602                .checked_sub(1)
18603                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18604            Ok(ReferenceInfo {
18605                first_sheet_index: Some(sheet_index),
18606                sheet_count: Some(1),
18607                first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18608            })
18609        };
18610
18611        let info = match reference {
18612            ReferenceType::Cell {
18613                sheet, row, col, ..
18614            } => {
18615                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18616                cell_info(sheet_name, *row, *col)?
18617            }
18618            ReferenceType::Range {
18619                sheet,
18620                start_row,
18621                start_col,
18622                ..
18623            } => {
18624                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18625                range_info(sheet_name, *start_row, *start_col)?
18626            }
18627            ReferenceType::Cell3D {
18628                sheet_first,
18629                sheet_last,
18630                row,
18631                col,
18632                ..
18633            } => {
18634                let first = cell_info(sheet_first, *row, *col)?;
18635                ReferenceInfo {
18636                    first_sheet_index: first.first_sheet_index,
18637                    sheet_count: self
18638                        .graph
18639                        .sheet_reg()
18640                        .active_span_len(sheet_first, sheet_last),
18641                    first_cell: first.first_cell,
18642                }
18643            }
18644            ReferenceType::Range3D {
18645                sheet_first,
18646                sheet_last,
18647                start_row,
18648                start_col,
18649                ..
18650            } => {
18651                let first = range_info(sheet_first, *start_row, *start_col)?;
18652                ReferenceInfo {
18653                    first_sheet_index: first.first_sheet_index,
18654                    sheet_count: self
18655                        .graph
18656                        .sheet_reg()
18657                        .active_span_len(sheet_first, sheet_last),
18658                    first_cell: first.first_cell,
18659                }
18660            }
18661            ReferenceType::NamedRange(name) => {
18662                let current_id = self
18663                    .graph
18664                    .sheet_id(current_sheet)
18665                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18666                let named = self
18667                    .graph
18668                    .resolve_name_entry(name, current_id)
18669                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18670                match &named.definition {
18671                    NamedDefinition::Cell(cell) => ReferenceInfo {
18672                        first_sheet_index: self
18673                            .graph
18674                            .sheet_reg()
18675                            .active_position_by_id(cell.sheet_id),
18676                        sheet_count: Some(1),
18677                        first_cell: Some(*cell),
18678                    },
18679                    NamedDefinition::Range(range) => ReferenceInfo {
18680                        first_sheet_index: self
18681                            .graph
18682                            .sheet_reg()
18683                            .active_position_by_id(range.start.sheet_id),
18684                        sheet_count: Some(1),
18685                        first_cell: Some(range.start),
18686                    },
18687                    NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
18688                        ReferenceInfo {
18689                            first_sheet_index: None,
18690                            sheet_count: None,
18691                            first_cell: None,
18692                        }
18693                    }
18694                }
18695            }
18696            ReferenceType::Table(tref) => {
18697                let table = self
18698                    .graph
18699                    .resolve_table_entry(&tref.name)
18700                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18701                ReferenceInfo {
18702                    first_sheet_index: self
18703                        .graph
18704                        .sheet_reg()
18705                        .active_position_by_id(table.range.start.sheet_id),
18706                    sheet_count: Some(1),
18707                    first_cell: Some(table.range.start),
18708                }
18709            }
18710            ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18711        };
18712
18713        Ok(Some(info))
18714    }
18715
18716    fn resolve_spill_reference(
18717        &self,
18718        anchor: &ReferenceType,
18719        current_sheet: &str,
18720    ) -> Result<ReferenceType, ExcelError> {
18721        let no_spill = || {
18722            ExcelError::new(ExcelErrorKind::Ref)
18723                .with_message("Spill reference anchor has no current spill")
18724        };
18725        // The rectangle keeps the operand's sheet qualification: an
18726        // unqualified anchor on the current sheet yields an unqualified range.
18727        let mut qualify = true;
18728        let anchor_cell = match anchor {
18729            ReferenceType::Cell {
18730                sheet, row, col, ..
18731            } => {
18732                qualify = sheet.is_some();
18733                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18734                let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(no_spill)?;
18735                if *row == 0 || *col == 0 {
18736                    return Err(no_spill());
18737                }
18738                CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true))
18739            }
18740            ReferenceType::NamedRange(name) => {
18741                let current_id = self.graph.sheet_id(current_sheet).ok_or_else(no_spill)?;
18742                let named = self
18743                    .graph
18744                    .resolve_name_entry(name, current_id)
18745                    .ok_or_else(no_spill)?;
18746                match &named.definition {
18747                    NamedDefinition::Cell(cell) => *cell,
18748                    NamedDefinition::Range(range)
18749                        if range.start.sheet_id == range.end.sheet_id
18750                            && range.start.coord.row() == range.end.coord.row()
18751                            && range.start.coord.col() == range.end.coord.col() =>
18752                    {
18753                        range.start
18754                    }
18755                    _ => {
18756                        return Err(ExcelError::new(ExcelErrorKind::Ref)
18757                            .with_message("Spill reference name must refer to a single cell"));
18758                    }
18759                }
18760            }
18761            _ => {
18762                return Err(ExcelError::new(ExcelErrorKind::Ref)
18763                    .with_message("Spill reference operand must be a single cell"));
18764            }
18765        };
18766        let vertex = self
18767            .graph
18768            .get_vertex_id_for_address(&anchor_cell)
18769            .ok_or_else(no_spill)?;
18770        // Policy: the spill operator applies only to dynamic arrays, never
18771        // to a legacy fixed-extent formula (including a single-cell one).
18772        if self.graph.fixed_single_arrays.contains(&vertex)
18773            || self.graph.fixed_array_shapes.contains_key(&vertex)
18774        {
18775            return Err(ExcelError::new(ExcelErrorKind::Ref)
18776                .with_message("Spill reference to a fixed array formula"));
18777        }
18778        let (first, last) = match self.graph.spill_extent_for_anchor(vertex) {
18779            Some(extent) => extent,
18780            // FORM211: a declared anchor (source spill identity) holding a
18781            // current scalar result is a 1x1 spill. A malformed registry
18782            // entry stays `#REF!`, as do blocked/error/empty results.
18783            None if !self.graph.spill_registry_has_anchor(vertex)
18784                && self.graph.is_current_declared_dynamic_anchor(vertex)
18785                && self.declared_anchor_holds_scalar(anchor_cell) =>
18786            {
18787                (anchor_cell, anchor_cell)
18788            }
18789            None => return Err(no_spill()),
18790        };
18791        let sheet_name = qualify.then(|| self.graph.sheet_name(first.sheet_id).to_string());
18792        let (sr, sc) = (first.coord.row() + 1, first.coord.col() + 1);
18793        let (er, ec) = (last.coord.row() + 1, last.coord.col() + 1);
18794        // The rectangle is concrete, so its bounds are absolute.
18795        Ok(if sr == er && sc == ec {
18796            ReferenceType::cell_with_abs(sheet_name, sr, sc, true, true)
18797        } else {
18798            ReferenceType::Range {
18799                sheet: sheet_name,
18800                start_row: Some(sr),
18801                start_col: Some(sc),
18802                end_row: Some(er),
18803                end_col: Some(ec),
18804                start_row_abs: true,
18805                start_col_abs: true,
18806                end_row_abs: true,
18807                end_col_abs: true,
18808            }
18809        })
18810    }
18811
18812    fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
18813        let sheet_name = self.graph.sheet_name(cell.sheet_id);
18814        if sheet_name.is_empty() {
18815            return Err(ExcelError::new(ExcelErrorKind::Ref));
18816        }
18817        let row = cell.coord.row() + 1;
18818        let col = cell.coord.col() + 1;
18819
18820        if let Some(entries) = self.staged_formulas.get(sheet_name)
18821            && let Some(text) = entries.get(row, col)
18822        {
18823            return Ok(Some(if text.starts_with('=') {
18824                text.to_owned()
18825            } else {
18826                format!("={text}")
18827            }));
18828        }
18829
18830        let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
18831            return Ok(None);
18832        };
18833        Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
18834    }
18835
18836    fn used_rows_for_columns(
18837        &self,
18838        sheet: &str,
18839        start_col: u32,
18840        end_col: u32,
18841    ) -> Option<(u32, u32)> {
18842        // Union Arrow-backed used-region with formula rows that have not been materialized yet.
18843        let sheet_id = self.graph.sheet_id(sheet)?;
18844        let snap = self.data_snapshot_id();
18845        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18846            guard
18847                .as_ref()
18848                .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
18849        }) {
18850            return cached;
18851        }
18852
18853        let arrow_bounds = self
18854            .sheet_store()
18855            .sheet(sheet)
18856            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18857        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18858        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18859            Some(bounds)
18860        } else {
18861            let sc0 = start_col.saturating_sub(1);
18862            let ec0 = end_col.saturating_sub(1);
18863            self.graph
18864                .used_row_bounds_for_columns(sheet_id, sc0, ec0)
18865                .map(|(a0, b0)| (a0 + 1, b0 + 1))
18866        };
18867
18868        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18869            guard
18870                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18871                .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
18872        }
18873
18874        computed
18875    }
18876
18877    fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
18878        // Union Arrow-backed used-region with formula columns that have not been materialized yet.
18879        let sheet_id = self.graph.sheet_id(sheet)?;
18880        let snap = self.data_snapshot_id();
18881        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18882            guard
18883                .as_ref()
18884                .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
18885        }) {
18886            return cached;
18887        }
18888
18889        let arrow_bounds = self
18890            .sheet_store()
18891            .sheet(sheet)
18892            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18893        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18894        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18895            Some(bounds)
18896        } else {
18897            let sr0 = start_row.saturating_sub(1);
18898            let er0 = end_row.saturating_sub(1);
18899            self.graph
18900                .used_col_bounds_for_rows(sheet_id, sr0, er0)
18901                .map(|(a0, b0)| (a0 + 1, b0 + 1))
18902        };
18903
18904        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18905            guard
18906                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18907                .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
18908        }
18909
18910        computed
18911    }
18912
18913    fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
18914        let _ = self.graph.sheet_id(sheet)?;
18915        // Excel-like upper bounds; we expose something finite but large.
18916        // Backends may override with real bounds.
18917        Some((1_048_576, 16_384)) // 1048576 rows, 16384 cols (XFD)
18918    }
18919
18920    fn data_snapshot_id(&self) -> u64 {
18921        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
18922    }
18923
18924    fn backend_caps(&self) -> crate::traits::BackendCaps {
18925        crate::traits::BackendCaps {
18926            streaming: true,
18927            used_region: true,
18928            write: false,
18929            tables: false,
18930            async_stream: false,
18931        }
18932    }
18933
18934    fn build_lookup_index(
18935        &self,
18936        view: &RangeView<'_>,
18937        axis: LookupAxis,
18938    ) -> Option<Arc<LookupIndex>> {
18939        self.build_lookup_index_impl(view, axis)
18940    }
18941
18942    // Flats removed
18943
18944    fn date_system(&self) -> crate::engine::DateSystem {
18945        self.config.date_system
18946    }
18947    /// New: resolve a reference into a RangeView (Phase 2 API)
18948    fn resolve_range_view<'c>(
18949        &'c self,
18950        reference: &ReferenceType,
18951        current_sheet: &str,
18952    ) -> Result<RangeView<'c>, ExcelError> {
18953        match reference {
18954            ReferenceType::External(ext) => {
18955                let name = ext.raw.as_str();
18956                match ext.kind {
18957                    formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
18958                        let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
18959                            return Err(ExcelError::new(ExcelErrorKind::Name)
18960                                .with_message(format!("Undefined name: {name}")));
18961                        };
18962                        let version = source
18963                            .version
18964                            .or_else(|| self.resolver.source_scalar_version(name));
18965                        let v = self.resolve_source_scalar_cached(name, version)?;
18966                        Ok(RangeView::from_owned_rows(
18967                            vec![vec![v]],
18968                            self.config.date_system,
18969                        ))
18970                    }
18971                    formualizer_parse::parser::ExternalRefKind::Range { .. } => {
18972                        let Some(source) = self.graph.resolve_source_table_entry(name) else {
18973                            // A deferred (whole-row/column) external range
18974                            // evaluates to #REF!, as it did before the parser
18975                            // kept it whole (see `unbound_external_range_defers`).
18976                            let kind =
18977                                if crate::engine::refs::unbound_external_range_defers(&ext.kind) {
18978                                    ExcelErrorKind::Ref
18979                                } else {
18980                                    ExcelErrorKind::Name
18981                                };
18982                            return Err(ExcelError::new(kind)
18983                                .with_message(format!("Undefined table: {name}")));
18984                        };
18985                        let version = source
18986                            .version
18987                            .or_else(|| self.resolver.source_table_version(name));
18988                        let key = (name.to_string(), version);
18989                        if let Ok(g) = self.source_cache.read()
18990                            && let Some(view) = g.external_views.get(&key)
18991                        {
18992                            return Ok(view.clone());
18993                        }
18994                        let table = self.resolve_source_table_cached(name, version)?;
18995                        let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
18996                        // Build under the write lock: concurrent readers of one
18997                        // source wait for a single materialization.
18998                        let Ok(mut g) = self.source_cache.write() else {
18999                            return self.source_table_to_range_view(table.as_ref(), &spec);
19000                        };
19001                        if let Some(view) = g.external_views.get(&key) {
19002                            return Ok(view.clone());
19003                        }
19004                        let view = self.source_table_to_range_view(table.as_ref(), &spec)?;
19005                        g.external_views.insert(key, view.clone());
19006                        Ok(view)
19007                    }
19008                }
19009            }
19010            ReferenceType::Range { .. } => {
19011                let shared = self.resolve_shared_ref(reference, current_sheet)?;
19012                let formualizer_common::SheetRef::Range(range) = shared else {
19013                    return Err(ExcelError::new(ExcelErrorKind::Ref));
19014                };
19015                // No context sheet is available here, so an unresolved locator
19016                // is #REF! rather than a guess (issue #110).
19017                let sheet_id = match range.sheet {
19018                    formualizer_common::SheetLocator::Id(id) => id,
19019                    formualizer_common::SheetLocator::Current
19020                    | formualizer_common::SheetLocator::Name(_) => {
19021                        return Err(ExcelError::new(ExcelErrorKind::Ref));
19022                    }
19023                };
19024                let bounded_range = if range.start_row.is_some()
19025                    && range.start_col.is_some()
19026                    && range.end_row.is_some()
19027                    && range.end_col.is_some()
19028                {
19029                    Some(RangeRef::try_from_shared(range.as_ref())?)
19030                } else {
19031                    None
19032                };
19033
19034                let sr = bounded_range
19035                    .as_ref()
19036                    .map(|r| r.start.coord.row() + 1)
19037                    .or_else(|| range.start_row.map(|b| b.index + 1));
19038                let sc = bounded_range
19039                    .as_ref()
19040                    .map(|r| r.start.coord.col() + 1)
19041                    .or_else(|| range.start_col.map(|b| b.index + 1));
19042                let er = bounded_range
19043                    .as_ref()
19044                    .map(|r| r.end.coord.row() + 1)
19045                    .or_else(|| range.end_row.map(|b| b.index + 1));
19046                let ec = bounded_range
19047                    .as_ref()
19048                    .map(|r| r.end.coord.col() + 1)
19049                    .or_else(|| range.end_col.map(|b| b.index + 1));
19050
19051                self.resolve_range_coordinates(sheet_id, sr, sc, er, ec)
19052            }
19053            ReferenceType::Cell { .. } => {
19054                let shared = self.resolve_shared_ref(reference, current_sheet)?;
19055                let formualizer_common::SheetRef::Cell(cell) = shared else {
19056                    return Err(ExcelError::new(ExcelErrorKind::Ref));
19057                };
19058                let addr = CellRef::try_from_shared(cell)?;
19059                let sheet_id = addr.sheet_id;
19060                let sheet_name = self.graph.sheet_name(sheet_id);
19061                let row = addr.coord.row() + 1;
19062                let col = addr.coord.col() + 1;
19063
19064                if self.force_materialize_range_views {
19065                    let v = self
19066                        .read_cell_value(sheet_name, row, col)
19067                        .unwrap_or(LiteralValue::Empty);
19068                    return Ok(RangeView::from_owned_rows(
19069                        vec![vec![v]],
19070                        self.config.date_system,
19071                    ));
19072                }
19073
19074                if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
19075                    let r0 = row.saturating_sub(1) as usize;
19076                    let c0 = col.saturating_sub(1) as usize;
19077                    let rv = asheet.range_view(r0, c0, r0, c0);
19078                    Ok(rv)
19079                } else {
19080                    let v = self
19081                        .read_cell_value(sheet_name, row, col)
19082                        .unwrap_or(LiteralValue::Empty);
19083                    Ok(RangeView::from_owned_rows(
19084                        vec![vec![v]],
19085                        self.config.date_system,
19086                    ))
19087                }
19088            }
19089            ReferenceType::NamedRange(name) => {
19090                if let Some(current_id) = self.graph.sheet_id(current_sheet)
19091                    && let Some(named) = self.graph.resolve_name_entry(name, current_id)
19092                {
19093                    match &named.definition {
19094                        NamedDefinition::Cell(cell_ref) => {
19095                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
19096                            if self.force_materialize_range_views {
19097                                let v = self
19098                                    .read_cell_value(
19099                                        sheet_name,
19100                                        cell_ref.coord.row() + 1,
19101                                        cell_ref.coord.col() + 1,
19102                                    )
19103                                    .unwrap_or(LiteralValue::Empty);
19104                                return Ok(RangeView::from_owned_rows(
19105                                    vec![vec![v]],
19106                                    self.config.date_system,
19107                                ));
19108                            } else {
19109                                let asheet = self
19110                                    .sheet_store()
19111                                    .sheet(sheet_name)
19112                                    .expect("Arrow sheet missing for named cell");
19113                                let r0 = cell_ref.coord.row() as usize;
19114                                let c0 = cell_ref.coord.col() as usize;
19115                                let rv = asheet.range_view(r0, c0, r0, c0);
19116                                return Ok(rv);
19117                            }
19118                        }
19119                        NamedDefinition::Range(range_ref) => {
19120                            let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
19121                            let sr = range_ref.start.coord.row() + 1;
19122                            let sc = range_ref.start.coord.col() + 1;
19123                            let er = range_ref.end.coord.row() + 1;
19124                            let ec = range_ref.end.coord.col() + 1;
19125                            if self.force_materialize_range_views {
19126                                let h = (er.saturating_sub(sr) + 1) as u64;
19127                                let w = (ec.saturating_sub(sc) + 1) as u64;
19128                                let cell_count = h.saturating_mul(w);
19129                                if cell_count <= self.config.spill.max_spill_cells as u64 {
19130                                    let mut rows: Vec<Vec<LiteralValue>> =
19131                                        Vec::with_capacity(h as usize);
19132                                    for r in sr..=er {
19133                                        let mut rowv: Vec<LiteralValue> =
19134                                            Vec::with_capacity(w as usize);
19135                                        for c in sc..=ec {
19136                                            rowv.push(
19137                                                self.read_cell_value(sheet_name, r, c)
19138                                                    .unwrap_or(LiteralValue::Empty),
19139                                            );
19140                                        }
19141                                        rows.push(rowv);
19142                                    }
19143                                    return Ok(RangeView::from_owned_rows(
19144                                        rows,
19145                                        self.config.date_system,
19146                                    ));
19147                                }
19148                            }
19149                            let asheet = self
19150                                .sheet_store()
19151                                .sheet(sheet_name)
19152                                .expect("Arrow sheet missing for named range");
19153                            let sr0 = range_ref.start.coord.row() as usize;
19154                            let sc0 = range_ref.start.coord.col() as usize;
19155                            let er0 = range_ref.end.coord.row() as usize;
19156                            let ec0 = range_ref.end.coord.col() as usize;
19157                            let rv = asheet.range_view(sr0, sc0, er0, ec0);
19158                            return Ok(rv);
19159                        }
19160                        NamedDefinition::Literal(v) => {
19161                            return Ok(RangeView::from_owned_rows(
19162                                vec![vec![v.clone()]],
19163                                self.config.date_system,
19164                            ));
19165                        }
19166                        NamedDefinition::Formula { ast, .. } => {
19167                            // A name that is exactly one external reference
19168                            // reads its source like the reference itself.
19169                            if let ASTNodeType::Reference {
19170                                reference: external @ ReferenceType::External(_),
19171                                ..
19172                            } = &ast.node_type
19173                            {
19174                                return self.resolve_range_view(external, current_sheet);
19175                            }
19176                            if let Some(value) = self.graph.get_value(named.vertex) {
19177                                return Ok(RangeView::from_owned_rows(
19178                                    vec![vec![value]],
19179                                    self.config.date_system,
19180                                ));
19181                            }
19182                        }
19183                    }
19184                }
19185
19186                if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
19187                    let version = source
19188                        .version
19189                        .or_else(|| self.resolver.source_scalar_version(name));
19190                    let v = self.resolve_source_scalar_cached(name, version)?;
19191                    return Ok(RangeView::from_owned_rows(
19192                        vec![vec![v]],
19193                        self.config.date_system,
19194                    ));
19195                }
19196
19197                let data = self.resolver.resolve_named_range_reference(name)?;
19198                Ok(RangeView::from_owned_rows(data, self.config.date_system))
19199            }
19200            ReferenceType::Table(tref) => {
19201                if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
19202                    let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
19203                    let asheet = self
19204                        .sheet_store()
19205                        .sheet(sheet_name)
19206                        .expect("Arrow sheet missing for table reference");
19207
19208                    let sr0 = table.range.start.coord.row() as usize;
19209                    let sc0 = table.range.start.coord.col() as usize;
19210                    let er0 = table.range.end.coord.row() as usize;
19211                    let ec0 = table.range.end.coord.col() as usize;
19212
19213                    let has_totals = table.totals_row;
19214                    let has_headers = table.header_row;
19215                    let data_sr = if has_headers {
19216                        sr0.saturating_add(1)
19217                    } else {
19218                        sr0
19219                    };
19220                    let data_er = if has_totals {
19221                        er0.saturating_sub(1)
19222                    } else {
19223                        er0
19224                    };
19225
19226                    let select = |sr: usize, sc: usize, er: usize, ec: usize| {
19227                        if sr > er || sc > ec {
19228                            asheet.range_view(1, 1, 0, 0)
19229                        } else {
19230                            asheet.range_view(sr, sc, er, ec)
19231                        }
19232                    };
19233
19234                    let av = match &tref.specifier {
19235                        None => {
19236                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19237                                "Table reference without specifier is unsupported".to_string(),
19238                            ));
19239                        }
19240                        Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
19241                            let Some(idx) = table.col_index(col) else {
19242                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19243                                    "Column refers to unknown table column".to_string(),
19244                                ));
19245                            };
19246                            let c0 = sc0 + idx;
19247                            select(data_sr, c0, data_er, c0)
19248                        }
19249                        Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
19250                            start,
19251                            end,
19252                        )) => {
19253                            let Some(si) = table.col_index(start) else {
19254                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19255                                    "Column range refers to unknown column(s)".to_string(),
19256                                ));
19257                            };
19258                            let Some(ei) = table.col_index(end) else {
19259                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19260                                    "Column range refers to unknown column(s)".to_string(),
19261                                ));
19262                            };
19263                            let (mut a, mut b) = (si, ei);
19264                            if a > b {
19265                                std::mem::swap(&mut a, &mut b);
19266                            }
19267                            let c_start = sc0 + a;
19268                            let c_end = sc0 + b;
19269                            select(data_sr, c_start, data_er, c_end)
19270                        }
19271                        Some(formualizer_parse::parser::TableSpecifier::All)
19272                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19273                            formualizer_parse::parser::SpecialItem::All,
19274                        )) => select(sr0, sc0, er0, ec0),
19275                        Some(formualizer_parse::parser::TableSpecifier::Data)
19276                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19277                            formualizer_parse::parser::SpecialItem::Data,
19278                        )) => select(data_sr, sc0, data_er, ec0),
19279                        Some(formualizer_parse::parser::TableSpecifier::Headers)
19280                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19281                            formualizer_parse::parser::SpecialItem::Headers,
19282                        )) => {
19283                            if !has_headers {
19284                                asheet.range_view(1, 1, 0, 0)
19285                            } else {
19286                                select(sr0, sc0, sr0, ec0)
19287                            }
19288                        }
19289                        Some(formualizer_parse::parser::TableSpecifier::Totals)
19290                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19291                            formualizer_parse::parser::SpecialItem::Totals,
19292                        )) => {
19293                            if !has_totals {
19294                                asheet.range_view(1, 1, 0, 0)
19295                            } else {
19296                                select(er0, sc0, er0, ec0)
19297                            }
19298                        }
19299                        Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19300                            formualizer_parse::parser::SpecialItem::ThisRow,
19301                        )) => {
19302                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19303                                "@ (This Row) requires table-aware context; not yet supported"
19304                                    .to_string(),
19305                            ));
19306                        }
19307                        Some(formualizer_parse::parser::TableSpecifier::Row(_))
19308                        | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
19309                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19310                                "Complex structured references not yet supported".to_string(),
19311                            ));
19312                        }
19313                    };
19314
19315                    return Ok(av);
19316                }
19317
19318                if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
19319                    let version = source
19320                        .version
19321                        .or_else(|| self.resolver.source_table_version(&tref.name));
19322                    let table = self.resolve_source_table_cached(&tref.name, version)?;
19323                    return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
19324                }
19325
19326                // Fallback: materialize via Resolver::resolve_range_like tranche 1.
19327                // A table nobody defines (an unbound reference kept by the
19328                // BestEffort preparation policy) is `#NAME?`, the kind Strict
19329                // reports at preparation, not the resolver's "not implemented".
19330                let boxed = self
19331                    .resolve_range_like(&ReferenceType::Table(tref.clone()))
19332                    .map_err(|e| {
19333                        if e.kind == ExcelErrorKind::NImpl {
19334                            ExcelError::new(ExcelErrorKind::Name)
19335                                .with_message(format!("Unknown table: {}", tref.name))
19336                        } else {
19337                            e
19338                        }
19339                    })?;
19340                let owned = boxed.materialise().into_owned();
19341                Ok(RangeView::from_owned_rows(owned, self.config.date_system))
19342            }
19343            ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
19344                Err(ExcelError::new(ExcelErrorKind::NImpl)
19345                    .with_message("3D references are not yet supported".to_string()))
19346            }
19347        }
19348    }
19349
19350    fn resolve_cell_format(
19351        &self,
19352        sheet: Option<&str>,
19353        row: u32,
19354        col: u32,
19355        current_sheet: &str,
19356    ) -> Option<crate::format::FormatId> {
19357        self.effective_format_id(sheet.unwrap_or(current_sheet), row, col)
19358    }
19359
19360    fn format_class(
19361        &self,
19362        format: crate::format::FormatId,
19363    ) -> Option<formualizer_common::numfmt::FormatClass> {
19364        self.format_registry.class(format).cloned()
19365    }
19366
19367    fn record_cell_derived_format(
19368        &self,
19369        sheet: &str,
19370        row: u32,
19371        col: u32,
19372        format: Option<crate::format::FormatId>,
19373    ) {
19374        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
19375            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
19376            self.record_derived_format_at(cell, format);
19377        }
19378    }
19379
19380    fn resolve_cell_reference_value(
19381        &self,
19382        sheet: Option<&str>,
19383        row: u32,
19384        col: u32,
19385        current_sheet: &str,
19386    ) -> Result<LiteralValue, ExcelError> {
19387        let sheet_name = sheet.unwrap_or(current_sheet);
19388        if self.graph.sheet_id(sheet_name).is_none() {
19389            return Err(ExcelError::new(ExcelErrorKind::Ref));
19390        }
19391        Ok(self
19392            .read_cell_value(sheet_name, row, col)
19393            .unwrap_or(LiteralValue::Empty))
19394    }
19395
19396    fn resolve_cell_reference_value_formatted(
19397        &self,
19398        sheet: Option<&str>,
19399        row: u32,
19400        col: u32,
19401        current_sheet: &str,
19402    ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
19403        // `resolve_cell_reference_value` + `resolve_cell_format` with one
19404        // sheet lookup of each kind.
19405        let sheet_name = sheet.unwrap_or(current_sheet);
19406        let Some(sheet_id) = self.graph.sheet_id(sheet_name) else {
19407            return Err(ExcelError::new(ExcelErrorKind::Ref));
19408        };
19409        let asheet = self.arrow_sheets.sheet(sheet_name);
19410        Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
19411    }
19412
19413    fn build_criteria_mask(
19414        &self,
19415        view: &RangeView<'_>,
19416        col_in_view: usize,
19417        pred: &crate::args::CriteriaPredicate,
19418    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19419        #[cfg(any(test, feature = "test-support"))]
19420        criteria_mask_test_hooks::note_mask(view.dims().0);
19421        if view.dims().1 == 0 {
19422            return None;
19423        }
19424        // If the view is logically open-ended but the backing sheet has no physical rows,
19425        // treat the mask as empty (0-len) rather than attempting to build a huge mask.
19426        let sheet_rows = view.sheet().nrows as usize;
19427        if sheet_rows == 0 || view.start_row() >= sheet_rows {
19428            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
19429        }
19430        self.criteria_mask_cache.get_or_build(
19431            view,
19432            col_in_view,
19433            pred,
19434            (
19435                self.recalc_epoch,
19436                self.topology_epoch,
19437                self.graph.topology_revision(),
19438            ),
19439            || compute_criteria_mask(view, col_in_view, pred),
19440        )
19441    }
19442
19443    fn build_row_visibility_mask(
19444        &self,
19445        view: &RangeView<'_>,
19446        mode: VisibilityMaskMode,
19447    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19448        self.build_row_visibility_mask_for_view(view, mode)
19449    }
19450
19451    fn nested_subtotal_cells(
19452        &self,
19453        view: &RangeView<'_>,
19454        include_aggregate: bool,
19455    ) -> Option<Vec<(usize, usize, usize)>> {
19456        self.nested_subtotal_cells_for_view(view, include_aggregate)
19457    }
19458}
19459
19460impl<R> Engine<R>
19461where
19462    R: EvaluationContext,
19463{
19464    fn clear_spill_projection_and_mirror(
19465        &mut self,
19466        anchor_vertex: VertexId,
19467        delta: Option<&mut DeltaCollector>,
19468    ) {
19469        let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
19470        if spill_cells.is_empty() {
19471            return;
19472        }
19473
19474        if let Some(delta) = delta
19475            && delta.mode != DeltaMode::Off
19476        {
19477            let empty = LiteralValue::Empty;
19478            for cell in spill_cells.iter() {
19479                let sheet_name = self.graph.sheet_name(cell.sheet_id);
19480                let old = self
19481                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19482                    .unwrap_or(LiteralValue::Empty);
19483                if old != empty {
19484                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19485                }
19486            }
19487        }
19488
19489        self.graph.clear_spill_region(anchor_vertex);
19490        if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
19491            self.record_structural_change(scope);
19492        }
19493
19494        if self.config.arrow_storage_enabled
19495            && self.config.delta_overlay_enabled
19496            && self.config.write_formula_overlay_enabled
19497        {
19498            let empty = LiteralValue::Empty;
19499            for cell in spill_cells.iter() {
19500                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19501                self.mirror_value_to_computed_overlay(
19502                    &sheet_name,
19503                    cell.coord.row() + 1,
19504                    cell.coord.col() + 1,
19505                    &empty,
19506                );
19507            }
19508        }
19509    }
19510
19511    /// Apply the evaluation outcome for one cyclic SCC: stamp `#CIRC!` on its
19512    /// (optionally filtered) members via `stamp_cycle_error`.
19513    ///
19514    /// This is the single per-SCC application point used by every schedule
19515    /// consumer walking `Schedule::units` (pre-work for #112, where cyclic
19516    /// SCCs will gain runtime verdicts instead of an unconditional stamp).
19517    ///
19518    /// `dirty_filter` preserves the recalc-plan quirk: when `Some(dirty)`,
19519    /// only members present in the set are stamped.
19520    ///
19521    /// Returns the number of vertices stamped (0 when a filter excludes every
19522    /// member), so callers can keep their site-specific `cycle_errors`
19523    /// accounting.
19524    fn apply_cycle_outcome(
19525        &mut self,
19526        cycle: &[VertexId],
19527        mut delta: Option<&mut DeltaCollector>,
19528        dirty_filter: Option<&FxHashSet<VertexId>>,
19529    ) -> usize {
19530        let circ_error = LiteralValue::Error(
19531            ExcelError::new(ExcelErrorKind::Circ)
19532                .with_message("Circular dependency detected".to_string()),
19533        );
19534        let mut stamped = 0usize;
19535        for &vertex_id in cycle {
19536            if let Some(filter) = dirty_filter
19537                && !filter.contains(&vertex_id)
19538            {
19539                continue;
19540            }
19541            self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
19542            stamped += 1;
19543        }
19544        stamped
19545    }
19546
19547    /// Stamp a vertex with `#CIRC!` as part of cycle handling.
19548    ///
19549    /// Unlike a bare `update_vertex_value`, this first tears down any spill the
19550    /// vertex previously anchored: it clears the spilled cells, releases the graph
19551    /// spill registry, drops any lingering region reservation, and mirrors the
19552    /// cleared cells into the computed overlay — the same teardown a normal scalar/
19553    /// error result performs (see `apply_non_array_result_from_parallel` /
19554    /// `clear_spill_projection_and_mirror`). Without this, a #CIRC stamp on a former
19555    /// spill anchor would leave stale spilled values and a reserved region behind
19556    /// (issue #111).
19557    ///
19558    /// When `delta` is provided, the cleared spill cells are recorded (by
19559    /// `clear_spill_projection_and_mirror`) and the anchor's own #CIRC change is
19560    /// recorded here, matching how other result paths emit deltas.
19561    fn stamp_cycle_error(
19562        &mut self,
19563        vertex_id: VertexId,
19564        circ_error: &LiteralValue,
19565        mut delta: Option<&mut DeltaCollector>,
19566    ) {
19567        // Tear down any previous spill projection/region before overwriting the anchor.
19568        if self.graph.spill_registry_has_anchor(vertex_id) {
19569            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
19570        }
19571        // Drop any reservation that was never committed (defensive; normally released
19572        // on the prior successful commit).
19573        self.spill_mgr.release_owner(vertex_id);
19574
19575        // Record the anchor's own #CIRC delta, like other result paths.
19576        if let Some(d) = delta
19577            && d.mode != DeltaMode::Off
19578            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
19579        {
19580            let sheet_name = self.graph.sheet_name(cell.sheet_id);
19581            let old = self
19582                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19583                .unwrap_or(LiteralValue::Empty);
19584            if old != *circ_error {
19585                d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19586            }
19587        }
19588
19589        self.graph.update_vertex_value_ref(vertex_id, circ_error);
19590        self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
19591    }
19592
19593    /// Dispatch point for one `ScheduleUnit::Cycle` (RFC #112, Stage 2).
19594    ///
19595    /// * `CycleDetection::Static` — today's behavior, byte-for-byte: stamp
19596    ///   `#CIRC!` on the (optionally dirty-filtered) members.
19597    /// * `CycleDetection::Runtime` — evaluate the SCC via
19598    ///   [`Self::evaluate_scc_unit`]. The recalc-plan dirty quirk maps to:
19599    ///   no dirty member → skip the task entirely (values stand); any dirty
19600    ///   member → the whole SCC evaluates (an SCC cannot be partially
19601    ///   evaluated).
19602    ///
19603    /// Returns the number of `#CIRC!`-stamped vertices, so call sites can
19604    /// keep their `cycle_errors` accounting (`> 0` ⇒ count the unit).
19605    fn handle_cycle_unit(
19606        &mut self,
19607        cycle: &[VertexId],
19608        mut delta: Option<&mut DeltaCollector>,
19609        dirty_filter: Option<&FxHashSet<VertexId>>,
19610        cancel_flag: Option<&AtomicBool>,
19611    ) -> Result<usize, ExcelError> {
19612        self.resource_checkpoint(cycle.len() as u64)?;
19613        match self.config.cycle.detection {
19614            CycleDetection::Static => {
19615                Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
19616            }
19617            CycleDetection::Runtime => {
19618                if let Some(filter) = dirty_filter
19619                    && !cycle.iter().any(|v| filter.contains(v))
19620                {
19621                    return Ok(0);
19622                }
19623                // Both policies share `evaluate_scc_unit`; they differ only
19624                // in the settle loop's live-cycle arm (Error stamps,
19625                // Iterate keeps passing — RFC #113).
19626                self.evaluate_scc_unit(cycle, delta, cancel_flag)
19627            }
19628        }
19629    }
19630
19631    /// Evaluate one statically-cyclic SCC under `CycleDetection::Runtime`
19632    /// (design doc `formualizer-stage2-scc-evaluation-design.md` §3; contract
19633    /// spec §3; Iterate policy arm per RFC #113).
19634    ///
19635    /// Phantom SCCs (live-acyclic) produce ordinary values under both
19636    /// policies; live cycles get `#CIRC!` with live-cycle-only blast radius
19637    /// under `CyclePolicy::Error`, or Excel-style iterative calculation
19638    /// (converge per spec §6 or cap at `max_iterations` passes) under
19639    /// `CyclePolicy::Iterate`. Runs sequentially on the
19640    /// coordinating thread; commits are write-through per member (no
19641    /// `ComputedWriteBuffer` — that buffer is scoped to layer evaluation and
19642    /// always flushed before a Cycle unit runs, G1), so later members' scalar
19643    /// *and* range reads observe earlier members' results through the overlay
19644    /// cascade. Deltas are recorded once per member at end of task (G11).
19645    ///
19646    /// Returns the number of vertices stamped `#CIRC!`.
19647    ///
19648    /// `pub(crate)` so tests can drive SCC shapes (e.g. name-vertex members)
19649    /// that ingest-time cycle rejection makes unreachable via public edits.
19650    pub(crate) fn evaluate_scc_unit(
19651        &mut self,
19652        cycle: &[VertexId],
19653        mut delta: Option<&mut DeltaCollector>,
19654        cancel_flag: Option<&AtomicBool>,
19655    ) -> Result<usize, ExcelError> {
19656        struct SccMember {
19657            vertex: VertexId,
19658            cell: Option<CellRef>,
19659        }
19660
19661        let task_start = crate::instant::FzInstant::now();
19662
19663        // ── 0. Member order (spec §7.13): cells ascending (sheet, row, col);
19664        // name vertices after, lexicographic by folded canonical name; any
19665        // other vertex kind (defensive — `get_evaluation_vertices` only emits
19666        // formula/name kinds) last by id, never evaluated.
19667        let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
19668        let mut name_members: Vec<(VertexId, String)> = Vec::new();
19669        let mut other_members: Vec<VertexId> = Vec::new();
19670        for &v in cycle {
19671            match self.graph.get_vertex_kind(v) {
19672                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
19673                    match self.graph.get_cell_ref(v) {
19674                        Some(cell) => cell_members.push((v, cell)),
19675                        None => other_members.push(v),
19676                    }
19677                }
19678                VertexKind::NamedScalar | VertexKind::NamedArray => {
19679                    match self.graph.name_key_for_vertex(v) {
19680                        Some(key) => name_members.push((v, key)),
19681                        None => other_members.push(v),
19682                    }
19683                }
19684                _ => other_members.push(v),
19685            }
19686        }
19687        cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
19688        name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
19689        other_members.sort_unstable();
19690
19691        let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
19692        let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
19693        let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
19694        for (v, c) in &cell_members {
19695            members.push(SccMember {
19696                vertex: *v,
19697                cell: Some(*c),
19698            });
19699        }
19700        for (v, _) in &name_members {
19701            members.push(SccMember {
19702                vertex: *v,
19703                cell: None,
19704            });
19705        }
19706        for v in &other_members {
19707            members.push(SccMember {
19708                vertex: *v,
19709                cell: None,
19710            });
19711        }
19712        let n = members.len();
19713        // Indices addressable by the collector (cells + names); `other`
19714        // members can be neither edge sources nor targets.
19715        let recordable = cell_refs.len() + name_keys.len();
19716
19717        let circ_error = LiteralValue::Error(
19718            ExcelError::new(ExcelErrorKind::Circ)
19719                .with_message("Circular dependency detected".to_string()),
19720        );
19721
19722        // ── 0b. Spec-§4 persistence repair: structural edits clear computed
19723        // overlays wholesale (`clear_computed_overlay_after_row/_col`), but
19724        // an iterating member's committed value is cycle STATE, not a
19725        // recomputable cache — and in canonical mode the overlay is its ONLY
19726        // home. If the overlay entry vanished since the last recalc, re-seed
19727        // it from the end-of-recalc snapshot (`iterative_state_values`) so
19728        // pass-1 reads (scalar AND range, via the overlay cascade) observe
19729        // the persisted value instead of silently restarting at Empty→0.
19730        // (Found by the iterate edge corpus: inserting/deleting an unrelated
19731        // row reset accumulators, violating spec §4/§7.15.)
19732        if !self.iterative_state_values.is_empty() {
19733            let restore: Vec<(VertexId, LiteralValue)> = members
19734                .iter()
19735                .filter_map(|m| {
19736                    let cell = m.cell?;
19737                    let persisted = self.iterative_state_values.get(&m.vertex)?;
19738                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
19739                    let overlay = self
19740                        .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19741                        .unwrap_or(LiteralValue::Empty);
19742                    if matches!(overlay, LiteralValue::Empty) {
19743                        Some((m.vertex, persisted.clone()))
19744                    } else {
19745                        None
19746                    }
19747                })
19748                .collect();
19749            for (vertex, value) in restore {
19750                self.mirror_vertex_value_to_overlay(vertex, &value);
19751            }
19752        }
19753
19754        // ── 1. Pre-task value snapshot (overlay-first for cells — G3; the
19755        // graph value map may be evicted in value-cache-disabled mode).
19756        let snapshot: Vec<LiteralValue> = members
19757            .iter()
19758            .map(|m| match m.cell {
19759                Some(cell) => {
19760                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
19761                    self.read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19762                        .unwrap_or(LiteralValue::Empty)
19763                }
19764                None => self
19765                    .graph
19766                    .get_value(m.vertex)
19767                    .unwrap_or(LiteralValue::Empty),
19768            })
19769            .collect();
19770
19771        // ── 2. Pre-scan: spill anchors (FormulaArray) are stamped `#CIRC!`
19772        // with full spill teardown (spec §7.9, #115) and excluded from
19773        // evaluation. They stay recordable edge TARGETS (readers see `#CIRC!`
19774        // and propagate). Non-evaluable defensive members are excluded too.
19775        let mut excluded = vec![false; n];
19776        let mut last_value = snapshot.clone();
19777        let mut stamped = 0usize;
19778        for (i, m) in members.iter().enumerate() {
19779            match self.graph.get_vertex_kind(m.vertex) {
19780                VertexKind::FormulaArray => {
19781                    // Deltas for the cleared spill-region cells (non-members)
19782                    // can only be recorded here; the anchor's own delta is
19783                    // covered by the end-of-task snapshot comparison (dedup).
19784                    self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
19785                    excluded[i] = true;
19786                    last_value[i] = circ_error.clone();
19787                    stamped += 1;
19788                }
19789                VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
19790                _ => excluded[i] = true,
19791            }
19792        }
19793
19794        let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
19795
19796        // Per-member live out-edges, refreshed whenever a member re-runs.
19797        let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
19798        // Position of each member in the most recent pass (-1 = did not run).
19799        let mut pos: Vec<i64> = vec![-1; n];
19800        // Whether each member's committed value changed in the most recent pass.
19801        let mut changed = vec![false; n];
19802
19803        // Evaluate-and-commit one member; returns Ok(true) when the member was
19804        // stamped `#CIRC!` (array result — would-be spill anchor, spec §7.9).
19805        macro_rules! run_member {
19806            ($i:expr) => {{
19807                let i: usize = $i;
19808                let m = &members[i];
19809                if i < recordable {
19810                    collector.set_current(i as u32);
19811                }
19812                let value = {
19813                    let ctx = RecordingContext::new(&*self, &collector);
19814                    match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
19815                        Ok(v) => v,
19816                        Err(e) => LiteralValue::Error(e),
19817                    }
19818                };
19819                let is_cell_formula = m.cell.is_some();
19820                if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
19821                    // A member that *would* spill inside an SCC gets the
19822                    // conservative §7.9 verdict. It has never spilled before
19823                    // (a prior spill would make it FormulaArray, pre-stamped
19824                    // above), so there is no projection to tear down.
19825                    self.stamp_cycle_error(m.vertex, &circ_error, None);
19826                    excluded[i] = true;
19827                    stamped += 1;
19828                    changed[i] = last_value[i] != circ_error;
19829                    last_value[i] = circ_error.clone();
19830                } else {
19831                    self.graph.update_vertex_value_ref(m.vertex, &value);
19832                    self.mirror_vertex_value_to_overlay(m.vertex, &value);
19833                    // §7.14 invariant (G2): a formula member must never be
19834                    // shadowed by a user/delta overlay entry, or iteration
19835                    // reads would silently diverge from committed values.
19836                    #[cfg(debug_assertions)]
19837                    if let Some(cell) = m.cell {
19838                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19839                        debug_assert!(
19840                            self.read_delta_overlay_cell(
19841                                &sheet_name,
19842                                cell.coord.row() + 1,
19843                                cell.coord.col() + 1
19844                            )
19845                            .is_none(),
19846                            "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
19847                            cell.coord.row() + 1,
19848                            cell.coord.col() + 1
19849                        );
19850                    }
19851                    changed[i] = last_value[i] != value;
19852                    last_value[i] = value;
19853                }
19854            }};
19855        }
19856
19857        let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
19858            if let Some(flag) = flag
19859                && flag.load(Ordering::Relaxed)
19860            {
19861                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
19862                    .with_message("Evaluation cancelled during SCC evaluation".to_string()));
19863            }
19864            Ok(())
19865        };
19866
19867        // ── 3. Pass 1: all evaluable members in member order.
19868        check_cancel(cancel_flag)?;
19869        let mut passes = 1usize;
19870        {
19871            let mut p = 0i64;
19872            for i in 0..n {
19873                if excluded[i] {
19874                    continue;
19875                }
19876                run_member!(i);
19877                pos[i] = p;
19878                p += 1;
19879            }
19880        }
19881
19882        // ── 4. Settle loop (design doc §3 step 4; RFC #113 policy arm).
19883        //
19884        // Acyclic classifications settle stale readers exactly (identical
19885        // under both policies — phantom SCCs never iterate). A witnessed
19886        // live cycle dispatches on policy: `Error` stamps `#CIRC!` and
19887        // stops; `Iterate` keeps running full passes over all members in
19888        // member order until converged (spec §6) or capped at
19889        // `max_iterations` total passes. A live cycle that only appears
19890        // mid-settle takes the same arm, and a cycle that dissolves
19891        // mid-iteration falls back to exact acyclic settling.
19892        //
19893        // Defensive acyclic budget: the acyclic settle is monotone, so more
19894        // than |SCC| + 2 settle passes can only be a bug; cap hits stamp the
19895        // remainder and set telemetry. Tracked via `settle_passes` so
19896        // iteration passes (legitimately many) don't consume the budget.
19897        let policy = self.config.cycle.policy;
19898        let cap = n + 2;
19899        let mut witnessed_cycles = 0usize;
19900        let mut capped = false;
19901        // ── Iterate-policy state ──
19902        let mut iterating = false;
19903        let mut converged = false;
19904        let mut exact_fixed_point = false;
19905        // Values committed by the last *full* pass; `None` until the first
19906        // iteration pass runs (pass 1 has no predecessor to compare against)
19907        // and reset when a settle pass runs (no cross-kind comparisons).
19908        let mut prev_pass: Option<Vec<LiteralValue>> = None;
19909        // Final-round convergence stats (overwritten per round so the values
19910        // reported are the ones observed at stop).
19911        let mut iter_max_delta = 0f64;
19912        let mut iter_nan_converged = 0usize;
19913        // Acyclic stale-reader re-eval passes (defensive budget; under pure
19914        // Error flow `1 + settle_passes == passes`, preserving Stage-2
19915        // behavior exactly).
19916        let mut settle_passes = 0usize;
19917        loop {
19918            // Drain this pass's recordings; members that ran replace their
19919            // out-edge set, members that didn't keep last-known edges.
19920            let drained = collector.take_edges();
19921            for i in 0..n {
19922                if pos[i] >= 0 {
19923                    out_edges[i].clear();
19924                }
19925            }
19926            for (from, to) in drained {
19927                debug_assert!(
19928                    pos[from as usize] >= 0,
19929                    "edge from a member that did not run"
19930                );
19931                out_edges[from as usize].push(to);
19932            }
19933            let mut edges: Vec<(u32, u32)> = Vec::new();
19934            for (i, outs) in out_edges.iter().enumerate() {
19935                if excluded[i] {
19936                    continue;
19937                }
19938                for &t in outs {
19939                    edges.push((i as u32, t));
19940                }
19941            }
19942            edges.sort_unstable();
19943            edges.dedup();
19944
19945            let analysis = analyze_live_graph(n, &edges);
19946
19947            if analysis.cycle_count > 0 {
19948                // Classification repeats every iteration pass under
19949                // `Iterate`; record the widest single witness instead of
19950                // accumulating so the count stays "distinct live cycles".
19951                witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
19952                match policy {
19953                    CyclePolicy::Error => {
19954                        // POLICY (Error): stamp every member of a live cycle,
19955                        // then one settling pass over the remaining members in
19956                        // live-topological order so error propagation
19957                        // downstream is consistent (spec §3.4). Blast radius =
19958                        // live cycles only.
19959                        for i in 0..n {
19960                            if analysis.in_cycle[i] && !excluded[i] {
19961                                self.stamp_cycle_error(members[i].vertex, &circ_error, None);
19962                                excluded[i] = true;
19963                                last_value[i] = circ_error.clone();
19964                                stamped += 1;
19965                            }
19966                        }
19967                        check_cancel(cancel_flag)?;
19968                        let order: Vec<usize> = analysis
19969                            .topo
19970                            .iter()
19971                            .map(|&i| i as usize)
19972                            .filter(|&i| !excluded[i])
19973                            .collect();
19974                        if !order.is_empty() {
19975                            passes += 1;
19976                            for i in order {
19977                                run_member!(i);
19978                            }
19979                        }
19980                        break;
19981                    }
19982                    CyclePolicy::Iterate {
19983                        max_iterations,
19984                        max_change,
19985                    } => {
19986                        // POLICY (Iterate), spec §3.5/§6.
19987                        iterating = true;
19988
19989                        // Convergence test: the full pass that just completed
19990                        // vs the previous full pass, per the spec-§6 rules.
19991                        // `prev_pass` is `None` until an iteration pass has
19992                        // run — pass 1 has no predecessor, so no convergence
19993                        // test occurs before the second pass (spec §6).
19994                        if let Some(prev) = &prev_pass {
19995                            let mut round_max_delta = 0f64;
19996                            let mut round_nan = 0usize;
19997                            let mut all_converged = true;
19998                            let mut round_exact = true;
19999                            for i in 0..n {
20000                                if excluded[i] {
20001                                    // Stamped mid-iteration (array result,
20002                                    // §7.9): the value is pinned and cannot
20003                                    // change again — trivially settled.
20004                                    continue;
20005                                }
20006                                let out = crate::engine::convergence::values_converged(
20007                                    &prev[i],
20008                                    &last_value[i],
20009                                    max_change,
20010                                    self.config.date_system,
20011                                );
20012                                if out.nan_converged {
20013                                    round_nan += 1;
20014                                }
20015                                if let Some(d) = out.abs_delta {
20016                                    round_max_delta = round_max_delta.max(d);
20017                                    if d != 0.0 {
20018                                        round_exact = false;
20019                                    }
20020                                }
20021                                if !out.converged {
20022                                    all_converged = false;
20023                                }
20024                            }
20025                            // Overwrite (not max): telemetry reports the
20026                            // round observed at stop.
20027                            iter_max_delta = round_max_delta;
20028                            iter_nan_converged = round_nan;
20029                            if all_converged {
20030                                exact_fixed_point = round_exact;
20031                                converged = true;
20032                                break;
20033                            }
20034                        }
20035
20036                        // ── Pass-counting reconciliation (spec §6/§7.6):
20037                        // `max_iterations` counts TOTAL passes, pass 1
20038                        // included, and pass 1 has already run by the time a
20039                        // live cycle is first witnessed here. The budget is
20040                        // therefore checked BEFORE evaluating anything more:
20041                        // with `max_iterations: 1` we stop right here — each
20042                        // member was evaluated exactly once this recalc (the
20043                        // Excel accumulator contract) and no convergence test
20044                        // ran (`prev_pass` is still `None`). Capping keeps
20045                        // the last committed values and is NOT an error
20046                        // (Excel parity); telemetry records it.
20047                        if passes >= max_iterations as usize {
20048                            capped = true;
20049                            break;
20050                        }
20051
20052                        check_cancel(cancel_flag)?;
20053                        // One more full pass over every evaluable member in
20054                        // member order (Gauss–Seidel: each commit is visible
20055                        // to later members within the pass). Live edges
20056                        // re-record — guards can flip near convergence
20057                        // (§7.3) — so classification repeats next time
20058                        // around, and a cycle that dissolves drops back to
20059                        // the exact acyclic settle below.
20060                        prev_pass = Some(last_value.clone());
20061                        for x in pos.iter_mut() {
20062                            *x = -1;
20063                        }
20064                        changed.fill(false);
20065                        passes += 1;
20066                        let mut p = 0i64;
20067                        for i in 0..n {
20068                            if excluded[i] {
20069                                continue;
20070                            }
20071                            run_member!(i);
20072                            pos[i] = p;
20073                            p += 1;
20074                        }
20075                        continue;
20076                    }
20077                }
20078            }
20079
20080            // Acyclic: find stale readers — members whose live read of `to`
20081            // happened before `to`'s value changed in the pass that just ran.
20082            let mut stale: Vec<usize> = Vec::new();
20083            for i in 0..n {
20084                if excluded[i] {
20085                    continue;
20086                }
20087                let is_stale = out_edges[i].iter().any(|&t| {
20088                    let t = t as usize;
20089                    changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
20090                });
20091                if is_stale {
20092                    stale.push(i);
20093                }
20094            }
20095            if stale.is_empty() {
20096                break; // values exact — phantom SCC (or dissolved live cycle)
20097            }
20098            if 1 + settle_passes >= cap {
20099                // Defensive only; hitting this is a bug (loud telemetry).
20100                capped = true;
20101                for (i, m) in members.iter().enumerate() {
20102                    if !excluded[i] {
20103                        self.stamp_cycle_error(m.vertex, &circ_error, None);
20104                        excluded[i] = true;
20105                        last_value[i] = circ_error.clone();
20106                        stamped += 1;
20107                    }
20108                }
20109                break;
20110            }
20111
20112            check_cancel(cancel_flag)?;
20113            // Re-evaluate stale readers in live-topo order, recording fresh
20114            // edges (branches may flip on re-eval — spec §7.3 — which is why
20115            // classification repeats).
20116            // A settle pass is a partial sweep: drop the full-pass baseline
20117            // so a live cycle (re)appearing afterwards never compares values
20118            // across mixed pass kinds.
20119            prev_pass = None;
20120            let topo_pos = analysis.topo_positions();
20121            stale.sort_unstable_by_key(|&i| topo_pos[i]);
20122            for x in pos.iter_mut() {
20123                *x = -1;
20124            }
20125            changed.fill(false);
20126            passes += 1;
20127            settle_passes += 1;
20128            for (p, i) in stale.into_iter().enumerate() {
20129                run_member!(i);
20130                pos[i] = p as i64;
20131            }
20132        }
20133
20134        // Post-work boundary: members are committed write-through, so a
20135        // member that ran across a live cancellation is already visible; the
20136        // task still must not complete (no delta, retention or iteration
20137        // state). The members stay dirty for the retry, as with the
20138        // mid-task checks above.
20139        self.live_cancellation_after_work(SCC_CANCELLED)?;
20140
20141        // Iteration that ended because the live cycle dissolved and the
20142        // acyclic settle reached exactness counts as converged (values are
20143        // exact, strictly better than threshold-converged). The defensive
20144        // settle cap (`capped` + stamping) is not.
20145        if iterating && !converged && !capped {
20146            converged = true;
20147        }
20148
20149        // ── 5. End of task: one delta per member whose final value differs
20150        // from the pre-task snapshot (spec §3 side-effect rule, G11).
20151        collector.clear_current();
20152        if let Some(d) = delta
20153            && d.mode != DeltaMode::Off
20154        {
20155            for (i, m) in members.iter().enumerate() {
20156                if let Some(cell) = m.cell
20157                    && last_value[i] != snapshot[i]
20158                {
20159                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20160                }
20161            }
20162        }
20163
20164        // Members of an SCC that iterated re-evaluate on EVERY recalc, like
20165        // Excel's circular cells: register them for the end-of-recalc
20166        // volatile-like redirty (see `pending_iterative_redirty`). Marking
20167        // any one member propagates around the (strongly connected) SCC and
20168        // to downstream dependents, but all members are registered so the
20169        // contract survives partial structural edits between recalcs.
20170        //
20171        // Exception (#368): an SCC that stopped on an
20172        // exact fixed point — every member reproduced its previous value
20173        // bit-for-bit before the pass cap, no NaN identity, no volatile or
20174        // dynamic-reference member — cannot change on a re-run with the same
20175        // inputs, so it is retained clean. The dirty graph decides when it
20176        // runs again. Tolerance-only convergence (|Δ| < max_change but ≠ 0),
20177        // capped SCCs (including the `max_iterations: 1` accumulator
20178        // contract) and volatile cycles keep the per-recalc redirty.
20179        // Whatever the outcome, this task supersedes any earlier retention
20180        // of its members.
20181        if !self.retained_scc_members.is_empty() {
20182            for m in members.iter() {
20183                self.retained_scc_members.remove(&m.vertex);
20184            }
20185        }
20186        if iterating {
20187            let retain = converged
20188                && !capped
20189                && exact_fixed_point
20190                && iter_nan_converged == 0
20191                && members
20192                    .iter()
20193                    .all(|m| !self.graph.is_volatile(m.vertex) && !self.graph.is_dynamic(m.vertex));
20194            if retain {
20195                if self.retained_scc_members.is_empty() {
20196                    self.retained_scc_config_fingerprint = self.retained_scc_config_fingerprint();
20197                    self.retained_scc_function_epoch_seen =
20198                        crate::function_registry::semantic_epoch();
20199                    self.retained_scc_provider_revision_seen =
20200                        self.resolver.planning_semantic_revision();
20201                }
20202                let scc_id = self.next_retained_scc_id;
20203                self.next_retained_scc_id = self.next_retained_scc_id.wrapping_add(1);
20204                for (i, m) in members.iter().enumerate() {
20205                    self.retained_scc_members.insert(m.vertex, scc_id);
20206                    // §4 persistence snapshot, written once: retained
20207                    // members do not pass through `redirty_for_next_recalc`.
20208                    if matches!(last_value[i], LiteralValue::Empty) {
20209                        self.iterative_state_values.remove(&m.vertex);
20210                    } else {
20211                        self.iterative_state_values
20212                            .insert(m.vertex, last_value[i].clone());
20213                    }
20214                }
20215            } else {
20216                self.pending_iterative_redirty
20217                    .extend(members.iter().map(|m| m.vertex));
20218            }
20219        } else if !self.iterative_state_values.is_empty() {
20220            // The cycle dissolved (phantom settle or `#CIRC!` stamping):
20221            // these members are ordinary formulas again and must not carry
20222            // stale iteration state into a future cycle.
20223            for m in members.iter() {
20224                self.iterative_state_values.remove(&m.vertex);
20225            }
20226        }
20227
20228        {
20229            let t = &mut self.last_cycle_telemetry;
20230            t.static_sccs += 1;
20231            if witnessed_cycles == 0 && stamped == 0 && !capped {
20232                t.phantom_sccs += 1;
20233            }
20234            t.live_cycles_witnessed += witnessed_cycles;
20235            t.circ_cells_stamped += stamped;
20236            t.settle_passes_total += passes;
20237            t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
20238            if iterating {
20239                t.iterated_sccs += 1;
20240                if converged {
20241                    t.converged_sccs += 1;
20242                }
20243                t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
20244                t.nan_converged += iter_nan_converged;
20245            }
20246            if capped {
20247                t.capped_sccs += 1;
20248            }
20249            t.elapsed_ms += task_start.elapsed().as_millis();
20250        }
20251
20252        Ok(stamped)
20253    }
20254
20255    /// Recorded sibling of [`Self::evaluate_vertex_immutable`]: evaluates one
20256    /// SCC member's AST via an [`Interpreter`] over a [`RecordingContext`] so
20257    /// reads that actually occur are captured as live edges. Value semantics
20258    /// must match `evaluate_vertex_immutable` exactly (including the missing-
20259    /// AST `Number(0.0)` quirk, G14); named Cell/Range/Literal definitions
20260    /// delegate to it after recording the definition region by hand (those
20261    /// reads bypass the context).
20262    fn evaluate_vertex_recorded(
20263        &self,
20264        vertex_id: VertexId,
20265        ctx: &RecordingContext<'_, R>,
20266        collector: &LiveEdgeCollector,
20267    ) -> Result<LiteralValue, ExcelError> {
20268        if !self.graph.vertex_exists(vertex_id) {
20269            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
20270                .with_message(format!("Vertex not found: {vertex_id:?}")));
20271        }
20272
20273        let kind = self.graph.get_vertex_kind(vertex_id);
20274        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
20275
20276        match kind {
20277            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20278                let Some(view) = self.graph.formula_view(vertex_id) else {
20279                    return Ok(LiteralValue::Number(0.0)); // G14 quirk
20280                };
20281                let sheet_name = self.graph.sheet_name(sheet_id);
20282                let cell_ref = self
20283                    .graph
20284                    .get_cell_ref(vertex_id)
20285                    .expect("cell ref for vertex");
20286                let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref)
20287                    .with_resolved_functions(&self.resolved_functions);
20288                interpreter
20289                    .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
20290                    .map(|cv| {
20291                        let format = cv.format_id();
20292                        self.record_derived_format(vertex_id, format);
20293                        self.materialize_formula_result(vertex_id, cv)
20294                    })
20295                    .or_else(|error| self.fit_formula_error(vertex_id, error))
20296            }
20297            VertexKind::NamedScalar | VertexKind::NamedArray => {
20298                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
20299                    ExcelError::new(ExcelErrorKind::Name)
20300                        .with_message("Named range metadata missing".to_string())
20301                })?;
20302
20303                match &named_range.definition {
20304                    NamedDefinition::Formula { ast, .. } => {
20305                        let context_sheet = match named_range.scope {
20306                            NameScope::Sheet(id) => id,
20307                            NameScope::Workbook => sheet_id,
20308                        };
20309                        let sheet_name = self.graph.sheet_name(context_sheet);
20310                        let cell_ref = self
20311                            .graph
20312                            .get_cell_ref(vertex_id)
20313                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
20314                        let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref)
20315                            .with_resolved_functions(&self.resolved_functions);
20316                        if kind == VertexKind::NamedScalar {
20317                            interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
20318                        } else {
20319                            match interpreter.evaluate_ast(ast) {
20320                                Ok(cv) => match cv.into_literal() {
20321                                    v @ LiteralValue::Array(_) => Ok(v),
20322                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
20323                                },
20324                                Err(err) => Ok(LiteralValue::Error(err)),
20325                            }
20326                        }
20327                    }
20328                    NamedDefinition::Cell(cell_ref) => {
20329                        // The definition is read via direct grid access in
20330                        // `evaluate_vertex_immutable`; record the live edge
20331                        // by hand before delegating.
20332                        collector.record_scalar(
20333                            cell_ref.sheet_id,
20334                            cell_ref.coord.row(),
20335                            cell_ref.coord.col(),
20336                        );
20337                        self.evaluate_vertex_immutable(vertex_id)
20338                    }
20339                    NamedDefinition::Range(range_ref) => {
20340                        if range_ref.start.sheet_id == range_ref.end.sheet_id {
20341                            collector.record_rect(
20342                                range_ref.start.sheet_id,
20343                                range_ref.start.coord.row(),
20344                                range_ref.start.coord.col(),
20345                                range_ref.end.coord.row(),
20346                                range_ref.end.coord.col(),
20347                            );
20348                        }
20349                        self.evaluate_vertex_immutable(vertex_id)
20350                    }
20351                    NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
20352                }
20353            }
20354            _ => self.evaluate_vertex_immutable(vertex_id),
20355        }
20356    }
20357
20358    /// Pending source occupancy is independent of formula preparation and value caches.
20359    fn pending_spill_occupied(&self, anchor: CellRef, end_row: u32, end_col: u32) -> bool {
20360        let sheet = self.graph.sheet_name(anchor.sheet_id);
20361        let package = self
20362            .staged_formulas
20363            .get(sheet)
20364            .and_then(|staged| staged.deferred_package.as_ref());
20365        self.staged_formula_index.occupies_spill(
20366            sheet,
20367            (anchor.coord.row() + 1, anchor.coord.col() + 1),
20368            (end_row + 1, end_col + 1),
20369            |point| package.is_some_and(|package| package.suppressed.contains(&point)),
20370        )
20371    }
20372
20373    fn remember_pending_spill(
20374        &mut self,
20375        vertex: VertexId,
20376        anchor: CellRef,
20377        region: Region,
20378    ) -> Result<(), ExcelError> {
20379        self.cancellation_checkpoint("pending spill occupancy")?;
20380        self.resource_checkpoint(1)?;
20381        if let Some(entry) = self
20382            .blocked_pending_spills
20383            .iter_mut()
20384            .find(|entry| entry.0 == vertex)
20385        {
20386            *entry = (vertex, anchor, region);
20387            return Ok(());
20388        }
20389        if self.blocked_pending_spills.len() == self.blocked_pending_spills.capacity() {
20390            // Geometric growth avoids copying every existing retry entry for
20391            // every new blocked anchor. Admit the entire capacity increment.
20392            let additional = self.blocked_pending_spills.capacity().max(1);
20393            let bytes =
20394                (additional as u64)
20395                    .saturating_mul(std::mem::size_of::<(VertexId, CellRef, Region)>() as u64);
20396            if let Some(ledger) = self.active_resource_ledger.as_mut() {
20397                ledger
20398                    .reserve_retained(bytes)
20399                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20400                self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
20401            }
20402            if self
20403                .blocked_pending_spills
20404                .try_reserve_exact(additional)
20405                .is_err()
20406            {
20407                if let Some(ledger) = self.active_resource_ledger.as_mut() {
20408                    ledger
20409                        .release_retained(bytes)
20410                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20411                    self.source_cache_accounted -= bytes;
20412                }
20413                return Err(crate::engine::ResourceLedgerError::Exhausted(
20414                    formualizer_common::ResourceExhaustionDetail {
20415                        reason: formualizer_common::ResourceExhaustionReason::RetainedMemory,
20416                        limit: u64::MAX,
20417                        observed: bytes,
20418                        request_id: None,
20419                    },
20420                )
20421                .into_excel_error());
20422            }
20423        }
20424        self.blocked_pending_spills.push((vertex, anchor, region));
20425        Ok(())
20426    }
20427
20428    // Successful edits wake only intersecting attempted regions. Keep the entry
20429    // until evaluation: logged edits may still roll back, and materializing a
20430    // pending formula must not destroy its anchor's occupancy retry information.
20431    fn invalidate_pending_spills(&mut self, scope: StructuralScope) {
20432        if let StructuralScope::RemovedSheet(sheet) = scope {
20433            self.blocked_pending_spills
20434                .retain(|entry| entry.1.sheet_id != sheet);
20435            return;
20436        }
20437        for &(vertex, anchor, region) in &self.blocked_pending_spills {
20438            let affected = match scope {
20439                StructuralScope::Cell { sheet, row, col } => {
20440                    region.intersects(&Region::point(sheet, row, col))
20441                }
20442                StructuralScope::Region(changed) => region.intersects(&changed),
20443                StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => {
20444                    region.sheet_id() == sheet
20445                }
20446                StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => true,
20447            };
20448            if affected
20449                && self.graph.vertex_exists(vertex)
20450                && self.graph.get_cell_ref(vertex) == Some(anchor)
20451                && matches!(
20452                    self.graph.get_vertex_kind(vertex),
20453                    VertexKind::FormulaScalar | VertexKind::FormulaArray
20454                )
20455            {
20456                self.graph.mark_vertex_dirty(vertex);
20457            }
20458        }
20459    }
20460
20461    fn guard_pending_spill_commit(
20462        &mut self,
20463        anchor_vertex: VertexId,
20464        targets: &[CellRef],
20465    ) -> Result<(), ExcelError> {
20466        let Some(anchor) = self.graph.get_cell_ref(anchor_vertex) else {
20467            return Ok(());
20468        };
20469        let Some(last) = targets.last() else {
20470            return Ok(());
20471        };
20472        let occupied = self.pending_spill_occupied(anchor, last.coord.row(), last.coord.col())
20473            || self
20474                .graph
20475                .table_intersects_spill(anchor, last.coord.row(), last.coord.col());
20476        if (occupied
20477            || self
20478                .blocked_pending_spills
20479                .iter()
20480                .any(|entry| entry.0 == anchor_vertex))
20481            && let Err(error) = self.remember_pending_spill(
20482                anchor_vertex,
20483                anchor,
20484                Region::rect(
20485                    anchor.sheet_id,
20486                    anchor.coord.row(),
20487                    last.coord.row(),
20488                    anchor.coord.col(),
20489                    last.coord.col(),
20490                ),
20491            )
20492        {
20493            self.spill_mgr.release_owner(anchor_vertex);
20494            return Err(error);
20495        }
20496        if occupied {
20497            self.spill_mgr.release_owner(anchor_vertex);
20498            return Err(ExcelError::new(ExcelErrorKind::Spill)
20499                .with_message("Spill blocked")
20500                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20501                    expected_rows: last.coord.row() - anchor.coord.row() + 1,
20502                    expected_cols: last.coord.col() - anchor.coord.col() + 1,
20503                }));
20504        }
20505        Ok(())
20506    }
20507
20508    /// Commit spill via shim and mirror resulting cells into Arrow overlay.
20509    fn commit_spill_and_mirror(
20510        &mut self,
20511        anchor_vertex: VertexId,
20512        targets: &[CellRef],
20513        rows: Vec<Vec<LiteralValue>>,
20514        delta: Option<&mut DeltaCollector>,
20515    ) -> Result<(), ExcelError> {
20516        self.guard_pending_spill_commit(anchor_vertex, targets)?;
20517        let prev_spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
20518
20519        if let Some(delta) = delta
20520            && delta.mode != DeltaMode::Off
20521        {
20522            let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20523                targets.iter().copied().collect();
20524            let empty = LiteralValue::Empty;
20525
20526            // Clears (prev - targets)
20527            for cell in prev_spill_cells.iter() {
20528                if target_set.contains(cell) {
20529                    continue;
20530                }
20531                let sheet_name = self.graph.sheet_name(cell.sheet_id);
20532                let old = self
20533                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20534                    .unwrap_or(LiteralValue::Empty);
20535                if old != empty {
20536                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20537                }
20538            }
20539
20540            // Writes (targets)
20541            if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
20542                let width = rows[0].len();
20543                for (idx, cell) in targets.iter().enumerate() {
20544                    let r_off = idx / width;
20545                    let c_off = idx % width;
20546                    let new = rows
20547                        .get(r_off)
20548                        .and_then(|r| r.get(c_off))
20549                        .cloned()
20550                        .unwrap_or(LiteralValue::Empty);
20551                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
20552                    let old = self
20553                        .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20554                        .unwrap_or(LiteralValue::Empty);
20555                    if old != new {
20556                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20557                    }
20558                }
20559            } else {
20560                // Degenerate shapes: if we have targets but no rows, treat as writing Empty.
20561                for cell in targets.iter() {
20562                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
20563                    let old = self
20564                        .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20565                        .unwrap_or(LiteralValue::Empty);
20566                    if !matches!(old, LiteralValue::Empty) {
20567                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20568                    }
20569                }
20570            }
20571        }
20572
20573        // Commit via shim (releases locks). When the graph value cache is disabled (Arrow-canonical
20574        // values), plan/commit must consult Arrow storage to detect non-empty value blockers.
20575        let arrow_sheets = &self.arrow_sheets;
20576        self.spill_mgr.commit_array_with_value_probe(
20577            &mut self.graph,
20578            anchor_vertex,
20579            targets,
20580            rows.clone(),
20581            |g, cell| {
20582                let sheet_name = g.sheet_name(cell.sheet_id);
20583                let asheet = arrow_sheets.sheet(sheet_name)?;
20584                let r0 = cell.coord.row() as usize;
20585                let c0 = cell.coord.col() as usize;
20586                let v = asheet.get_cell_value(r0, c0);
20587                if matches!(v, LiteralValue::Empty) {
20588                    None
20589                } else {
20590                    Some(v)
20591                }
20592            },
20593        )?;
20594
20595        self.blocked_pending_spills
20596            .retain(|entry| entry.0 != anchor_vertex);
20597        self.spill_blocker_waiters.forget(anchor_vertex);
20598        if let Some(scope) = Self::structural_scope_from_cells(&prev_spill_cells) {
20599            self.record_structural_change(scope);
20600        }
20601        if let Some(scope) = Self::structural_scope_from_cells(targets) {
20602            self.record_structural_change(scope);
20603        }
20604
20605        if self.config.arrow_storage_enabled
20606            && self.config.delta_overlay_enabled
20607            && self.config.write_formula_overlay_enabled
20608        {
20609            if !prev_spill_cells.is_empty() {
20610                let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20611                    targets.iter().copied().collect();
20612                let empty = LiteralValue::Empty;
20613                for cell in prev_spill_cells.iter() {
20614                    if !target_set.contains(cell) {
20615                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20616                        self.mirror_value_to_computed_overlay(
20617                            &sheet_name,
20618                            cell.coord.row() + 1,
20619                            cell.coord.col() + 1,
20620                            &empty,
20621                        );
20622                    }
20623                }
20624            }
20625
20626            for (idx, cell) in targets.iter().enumerate() {
20627                if rows.is_empty() || rows[0].is_empty() {
20628                    break;
20629                }
20630                let width = rows[0].len();
20631                let r_off = idx / width;
20632                let c_off = idx % width;
20633                let v = rows[r_off][c_off].clone();
20634                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20635                self.mirror_value_to_computed_overlay(
20636                    &sheet_name,
20637                    cell.coord.row() + 1,
20638                    cell.coord.col() + 1,
20639                    &v,
20640                );
20641            }
20642        }
20643        Ok(())
20644    }
20645}
20646
20647#[cfg(test)]
20648#[path = "tests/authority_schedule_execution.rs"]
20649mod authority_schedule_execution;
20650
20651#[cfg(test)]
20652#[path = "tests/scalar_commit.rs"]
20653mod scalar_commit_tests;
20654
20655#[cfg(test)]
20656#[path = "tests/pending_spill.rs"]
20657mod pending_spill_tests;
20658
20659#[cfg(test)]
20660#[path = "tests/spill_batch_abort_192.rs"]
20661mod spill_batch_abort_192;
20662
20663// ── Effects pipeline (ticket 603) ──────────────────────────────────────────
20664//
20665// Compute → Plan → Apply separation for evaluation side-effects.
20666
20667use crate::engine::effects::Effect;
20668
20669// Scalar writes and spill clears stay inline; larger spill plans grow normally.
20670type PlannedEffects = smallvec::SmallVec<[Effect; 2]>;
20671use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
20672
20673impl<R> Engine<R>
20674where
20675    R: EvaluationContext,
20676{
20677    /// Plan effects for a single vertex after its value has been computed.
20678    ///
20679    /// This reads graph state but only performs lightweight mutations
20680    /// (`set_kind`, `spill_mgr.reserve`) that are needed for correctness
20681    /// during the planning phase.  Value-changing mutations are deferred to
20682    /// `apply_effect`.
20683    pub(crate) fn plan_vertex_effects(
20684        &mut self,
20685        vertex_id: VertexId,
20686        computed_value: LiteralValue,
20687    ) -> Result<PlannedEffects, ExcelError> {
20688        // FR3: a stale dynamic reader publishes nothing; FR2: anything else
20689        // leaves the dirty set at its commit (design §8.2).
20690        if self.freshness_armed() {
20691            if self.freshness_drop_stale(vertex_id) {
20692                return Ok(PlannedEffects::new());
20693            }
20694            let effects = self.plan_vertex_effects_unrecorded(vertex_id, computed_value)?;
20695            self.freshness_mark_committed(vertex_id);
20696            return Ok(effects);
20697        }
20698        self.plan_vertex_effects_unrecorded(vertex_id, computed_value)
20699    }
20700
20701    fn plan_vertex_effects_unrecorded(
20702        &mut self,
20703        vertex_id: VertexId,
20704        computed_value: LiteralValue,
20705    ) -> Result<PlannedEffects, ExcelError> {
20706        let kind = self.graph.get_vertex_kind(vertex_id);
20707        let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
20708
20709        // If this vertex's cell is currently covered by a spill from a different
20710        // anchor, ignore the computed result.  Formula vertices are exempt:
20711        // they must still evaluate so that overlapping spills produce #SPILL!.
20712        if !is_formula {
20713            if let Some(cell) = self.graph.get_cell_ref(vertex_id)
20714                && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
20715                && owner != vertex_id
20716            {
20717                return Ok(PlannedEffects::new());
20718            }
20719            // Non-formula vertices: store value as-is (arrays remain arrays; no spill).
20720            return Ok(smallvec::smallvec![Effect::WriteCell {
20721                vertex_id,
20722                value: computed_value,
20723            }]);
20724        }
20725
20726        match computed_value {
20727            LiteralValue::Array(rows) => self.plan_array_effects(vertex_id, rows),
20728            other => self.plan_scalar_effects(vertex_id, other),
20729        }
20730    }
20731
20732    /// Plan effects for a formula vertex that produced a scalar/error result.
20733    fn plan_scalar_effects(
20734        &mut self,
20735        vertex_id: VertexId,
20736        value: LiteralValue,
20737    ) -> Result<PlannedEffects, ExcelError> {
20738        // Range admission substitutes the same cap error before allocating rows.
20739        // Preserve the owned-array rejection's kind, release and clear effects.
20740        if let LiteralValue::Error(error) = &value
20741            && error.kind == ExcelErrorKind::Spill
20742            && error.message() == Some("SpillTooLarge")
20743            && let formualizer_common::ExcelErrorExtra::Spill {
20744                expected_rows,
20745                expected_cols,
20746            } = error.extra()
20747            && u64::from(*expected_rows).saturating_mul(u64::from(*expected_cols))
20748                > u64::from(self.config.spill.max_spill_cells)
20749        {
20750            self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20751            return self.plan_spill_error_effects(
20752                vertex_id,
20753                "SpillTooLarge",
20754                *expected_rows,
20755                *expected_cols,
20756            );
20757        }
20758        if !matches!(&value, LiteralValue::Error(e) if e.kind == ExcelErrorKind::Spill) {
20759            self.blocked_pending_spills
20760                .retain(|entry| entry.0 != vertex_id);
20761            self.spill_blocker_waiters.forget(vertex_id);
20762        }
20763        let has_spill = self
20764            .graph
20765            .spill_cells_for_anchor(vertex_id)
20766            .is_some_and(|c| !c.is_empty());
20767
20768        let mut effects = PlannedEffects::new();
20769        if has_spill {
20770            effects.push(Effect::SpillClear {
20771                anchor_vertex: vertex_id,
20772            });
20773        }
20774        effects.push(Effect::WriteCell { vertex_id, value });
20775        Ok(effects)
20776    }
20777
20778    /// Plan effects for a formula vertex that produced an array result.
20779    fn plan_array_effects(
20780        &mut self,
20781        vertex_id: VertexId,
20782        rows: Vec<Vec<LiteralValue>>,
20783    ) -> Result<PlannedEffects, ExcelError> {
20784        // Lightweight mutation needed for correct spill-blocking checks.
20785        self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20786
20787        let anchor = self
20788            .graph
20789            .get_cell_ref(vertex_id)
20790            .expect("cell ref for vertex");
20791        let sheet_id = anchor.sheet_id;
20792        let h = rows.len() as u32;
20793        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
20794
20795        // Hard cap to avoid vertex explosion from huge dynamic arrays.
20796        let spill_cells = (h as u64).saturating_mul(w as u64);
20797        if spill_cells > self.config.spill.max_spill_cells as u64 {
20798            return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
20799        }
20800
20801        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity).
20802        const PACKED_MAX_ROW: u32 = 1_048_575;
20803        const PACKED_MAX_COL: u32 = 16_383;
20804        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
20805        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
20806        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
20807            return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
20808        }
20809
20810        if h != 0 && w != 0 && self.graph.table_intersects_spill(anchor, end_row, end_col) {
20811            return self.plan_spill_error_effects(vertex_id, "Spill blocked by table", h, w);
20812        }
20813        let mut targets = Vec::new();
20814        for r in 0..h {
20815            for c in 0..w {
20816                targets.push(self.graph.make_cell_ref_internal(
20817                    sheet_id,
20818                    anchor.coord.row() + r,
20819                    anchor.coord.col() + c,
20820                ));
20821            }
20822        }
20823
20824        if h != 0 && w != 0 {
20825            let occupied = self.pending_spill_occupied(anchor, end_row, end_col);
20826            if occupied
20827                || self
20828                    .blocked_pending_spills
20829                    .iter()
20830                    .any(|entry| entry.0 == vertex_id)
20831            {
20832                self.spill_mgr.release_owner(vertex_id);
20833                self.remember_pending_spill(
20834                    vertex_id,
20835                    anchor,
20836                    Region::rect(
20837                        sheet_id,
20838                        anchor.coord.row(),
20839                        end_row,
20840                        anchor.coord.col(),
20841                        end_col,
20842                    ),
20843                )?;
20844            }
20845            if occupied {
20846                return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20847            }
20848        }
20849
20850        // Region lock via spill manager.
20851        match self.spill_mgr.reserve(
20852            vertex_id,
20853            anchor,
20854            SpillShape { rows: h, cols: w },
20855            SpillMeta {
20856                epoch: self.recalc_epoch,
20857                config: self.config.spill,
20858            },
20859        ) {
20860            Ok(()) => {
20861                // Validate spill region is available.
20862                let mut yield_effects = PlannedEffects::new();
20863                if let Err((e, blocker)) = self
20864                    .graph
20865                    .plan_spill_region_with_blocker(vertex_id, &targets)
20866                {
20867                    let yielding = if e.message() == Some("BlockedBySpill") {
20868                        self.graph.spills_yielding_to(vertex_id, &targets)
20869                    } else {
20870                        None
20871                    };
20872                    if let Some(yielding) = yielding {
20873                        // A colliding spill later in (sheet, column, row)
20874                        // order yields: it is cleared and replans to
20875                        // `#SPILL!` with its dependents.
20876                        for &other in &yielding {
20877                            let (rows, cols) = self
20878                                .graph
20879                                .spill_extent_for_anchor(other)
20880                                .map(|(first, last)| {
20881                                    (
20882                                        last.coord.row() - first.coord.row() + 1,
20883                                        last.coord.col() - first.coord.col() + 1,
20884                                    )
20885                                })
20886                                .unwrap_or((0, 0));
20887                            yield_effects.push(Effect::SpillClear {
20888                                anchor_vertex: other,
20889                            });
20890                            yield_effects.push(Effect::WriteCell {
20891                                vertex_id: other,
20892                                value: Self::spill_error_value("Spill blocked", rows, cols),
20893                            });
20894                        }
20895                        self.graph.mark_dirty_many(&yielding);
20896                    } else {
20897                        // A formula or spill blocker can go away without this
20898                        // anchor recomputing: remember the attempted region so
20899                        // an edit inside it wakes the anchor.
20900                        if matches!(e.message(), Some("BlockedByFormula" | "BlockedBySpill")) {
20901                            self.spill_blocker_waiters.wait(vertex_id, blocker);
20902                        }
20903                        return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20904                    }
20905                }
20906
20907                // Arrow-canonical mode: graph planning cannot see non-empty value blockers because
20908                // cell values are not cached in the dependency graph. Consult Arrow storage to
20909                // detect occupied cells in the target region.
20910                if !self.graph.value_cache_enabled() {
20911                    let sheet_name = self.graph.sheet_name(sheet_id);
20912                    if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
20913                        for cell in targets.iter() {
20914                            // Allow overwriting the anchor itself.
20915                            if *cell == anchor {
20916                                continue;
20917                            }
20918                            // Allow cells already owned by a spill (plan() validated spill ownership).
20919                            if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
20920                                continue;
20921                            }
20922                            // Skip formula blockers; plan() handled them (or allowed).
20923                            if let Some(vid) = self.graph.get_vertex_id_for_address(cell)
20924                                && vid != vertex_id
20925                            {
20926                                match self.graph.get_vertex_kind(vid) {
20927                                    VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20928                                        continue;
20929                                    }
20930                                    _ => {}
20931                                }
20932                            }
20933
20934                            let v = asheet.get_cell_value(
20935                                cell.coord.row() as usize,
20936                                cell.coord.col() as usize,
20937                            );
20938                            if !matches!(v, LiteralValue::Empty) {
20939                                return self.plan_spill_error_effects(
20940                                    vertex_id,
20941                                    "BlockedByValue",
20942                                    h,
20943                                    w,
20944                                );
20945                            }
20946                        }
20947                    }
20948                }
20949
20950                let top_left = rows
20951                    .first()
20952                    .and_then(|r| r.first())
20953                    .cloned()
20954                    .unwrap_or(LiteralValue::Empty);
20955
20956                let mut effects = yield_effects;
20957                // Clear previous spill if any.
20958                let has_prev = self
20959                    .graph
20960                    .spill_cells_for_anchor(vertex_id)
20961                    .is_some_and(|c| !c.is_empty());
20962                if has_prev {
20963                    effects.push(Effect::SpillClear {
20964                        anchor_vertex: vertex_id,
20965                    });
20966                }
20967                effects.push(Effect::SpillCommit {
20968                    anchor_vertex: vertex_id,
20969                    anchor_cell: anchor,
20970                    target_cells: targets,
20971                    values: rows,
20972                });
20973                effects.push(Effect::WriteCell {
20974                    vertex_id,
20975                    value: top_left,
20976                });
20977                Ok(effects)
20978            }
20979            Err(e) => {
20980                let msg = e.message().unwrap_or("Spill blocked");
20981                self.plan_spill_error_effects(vertex_id, msg, h, w)
20982            }
20983        }
20984    }
20985
20986    /// The `#SPILL!` value a blocked anchor holds.
20987    fn spill_error_value(message: &str, expected_rows: u32, expected_cols: u32) -> LiteralValue {
20988        LiteralValue::Error(
20989            ExcelError::new(ExcelErrorKind::Spill)
20990                .with_message(message)
20991                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20992                    expected_rows,
20993                    expected_cols,
20994                }),
20995        )
20996    }
20997
20998    /// Build the effect list for a spill that failed validation.
20999    fn plan_spill_error_effects(
21000        &mut self,
21001        vertex_id: VertexId,
21002        message: &str,
21003        expected_rows: u32,
21004        expected_cols: u32,
21005    ) -> Result<PlannedEffects, ExcelError> {
21006        self.spill_mgr.release_owner(vertex_id);
21007        let spill_val = Self::spill_error_value(message, expected_rows, expected_cols);
21008
21009        let effects = smallvec::smallvec![
21010            Effect::SpillClear {
21011                anchor_vertex: vertex_id,
21012            },
21013            Effect::WriteCell {
21014                vertex_id,
21015                value: spill_val,
21016            },
21017        ];
21018        Ok(effects)
21019    }
21020
21021    /// Apply a single effect, performing the actual graph mutations.
21022    pub(crate) fn apply_effect(
21023        &mut self,
21024        effect: &Effect,
21025        delta: Option<&mut DeltaCollector>,
21026        log: Option<&mut ChangeLog>,
21027    ) -> Result<(), ExcelError> {
21028        self.apply_effect_with_computed_writes(effect, delta, log, None)
21029    }
21030
21031    fn apply_effect_with_computed_writes(
21032        &mut self,
21033        effect: &Effect,
21034        delta: Option<&mut DeltaCollector>,
21035        log: Option<&mut ChangeLog>,
21036        computed_writes: Option<&mut ComputedWriteBuffer>,
21037    ) -> Result<(), ExcelError> {
21038        match effect {
21039            Effect::WriteCell { vertex_id, value } => {
21040                self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
21041            }
21042            Effect::SpillClear { anchor_vertex } => {
21043                self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
21044            }
21045            Effect::SpillCommit {
21046                anchor_vertex,
21047                anchor_cell: _,
21048                target_cells,
21049                values,
21050            } => {
21051                self.apply_spill_commit(
21052                    *anchor_vertex,
21053                    target_cells,
21054                    values.clone(),
21055                    delta,
21056                    log,
21057                    computed_writes,
21058                )?;
21059            }
21060        }
21061        Ok(())
21062    }
21063
21064    /// Apply a WriteCell effect.
21065    fn apply_write_cell(
21066        &mut self,
21067        vertex_id: VertexId,
21068        value: &LiteralValue,
21069        delta: Option<&mut DeltaCollector>,
21070        mut computed_writes: Option<&mut ComputedWriteBuffer>,
21071    ) -> Result<(), ExcelError> {
21072        if let Some(d) = delta
21073            && d.mode != DeltaMode::Off
21074        {
21075            if let Some(buffer) = computed_writes.as_deref_mut() {
21076                self.flush_computed_write_buffer(buffer)?;
21077            }
21078            if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
21079                let sheet_name = self.graph.sheet_name(cell.sheet_id);
21080                let old = self
21081                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21082                    .unwrap_or(LiteralValue::Empty);
21083                if old != *value {
21084                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
21085                }
21086            }
21087        }
21088        self.graph.update_vertex_value_ref(vertex_id, value);
21089        self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
21090        Ok(())
21091    }
21092
21093    /// Apply a SpillClear effect.
21094    fn apply_spill_clear(
21095        &mut self,
21096        anchor_vertex: VertexId,
21097        delta: Option<&mut DeltaCollector>,
21098        log: Option<&mut ChangeLog>,
21099        computed_writes: Option<&mut ComputedWriteBuffer>,
21100    ) -> Result<(), ExcelError> {
21101        if let Some(buffer) = computed_writes {
21102            self.flush_computed_write_buffer(buffer)?;
21103        }
21104
21105        let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
21106        if spill_cells.is_empty() {
21107            return Ok(());
21108        }
21109
21110        // Snapshot for ChangeLog before clearing.
21111        let snapshot = if log.is_some() {
21112            self.snapshot_spill_for_anchor(anchor_vertex)
21113        } else {
21114            None
21115        };
21116
21117        // Record delta for cleared cells.
21118        if let Some(d) = delta
21119            && d.mode != DeltaMode::Off
21120        {
21121            let empty = LiteralValue::Empty;
21122            for cell in spill_cells.iter() {
21123                let sheet_name = self.graph.sheet_name(cell.sheet_id);
21124                let old = self
21125                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21126                    .unwrap_or(LiteralValue::Empty);
21127                if old != empty {
21128                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
21129                }
21130            }
21131        }
21132
21133        self.graph.clear_spill_region(anchor_vertex);
21134        if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
21135            self.record_structural_change(scope);
21136        }
21137
21138        // Mirror Empty to Arrow overlay for cleared cells.
21139        if self.config.arrow_storage_enabled
21140            && self.config.delta_overlay_enabled
21141            && self.config.write_formula_overlay_enabled
21142        {
21143            let empty = LiteralValue::Empty;
21144            for cell in spill_cells.iter() {
21145                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
21146                self.mirror_value_to_computed_overlay(
21147                    &sheet_name,
21148                    cell.coord.row() + 1,
21149                    cell.coord.col() + 1,
21150                    &empty,
21151                );
21152            }
21153        }
21154
21155        // ChangeLog.
21156        if let Some(log) = log
21157            && let Some(old) = snapshot
21158        {
21159            log.record(ChangeEvent::SpillCleared {
21160                anchor: anchor_vertex,
21161                old,
21162            });
21163        }
21164        Ok(())
21165    }
21166
21167    /// Apply a SpillCommit effect.
21168    fn apply_spill_commit(
21169        &mut self,
21170        anchor_vertex: VertexId,
21171        target_cells: &[CellRef],
21172        values: Vec<Vec<LiteralValue>>,
21173        delta: Option<&mut DeltaCollector>,
21174        log: Option<&mut ChangeLog>,
21175        computed_writes: Option<&mut ComputedWriteBuffer>,
21176    ) -> Result<(), ExcelError> {
21177        self.guard_pending_spill_commit(anchor_vertex, target_cells)?;
21178        if let Some(buffer) = computed_writes {
21179            self.flush_computed_write_buffer(buffer)?;
21180        }
21181
21182        // Snapshot for ChangeLog before commit.
21183        let old_snapshot = if log.is_some() {
21184            self.snapshot_spill_for_anchor(anchor_vertex)
21185        } else {
21186            None
21187        };
21188
21189        // Delegate to existing commit_spill_and_mirror for delta + overlay logic.
21190        self.commit_spill_and_mirror(anchor_vertex, target_cells, values.clone(), delta)?;
21191
21192        // ChangeLog.
21193        if let Some(log) = log {
21194            log.record(ChangeEvent::SpillCommitted {
21195                anchor: anchor_vertex,
21196                old: old_snapshot,
21197                new: SpillSnapshot {
21198                    target_cells: target_cells.to_vec(),
21199                    values,
21200                },
21201            });
21202        }
21203        Ok(())
21204    }
21205
21206    /// Snapshot a spill region for ChangeLog recording.
21207    ///
21208    /// Extracted from `VertexEditor::snapshot_spill_for_anchor` to be usable
21209    /// without creating a `VertexEditor`.
21210    fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
21211        let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
21212        if cells.is_empty() {
21213            return None;
21214        }
21215
21216        let max = self.config.spill.max_spill_cells as usize;
21217        let mut cells = cells;
21218        if cells.len() > max {
21219            cells.truncate(max);
21220        }
21221
21222        let first = *cells.first().expect("non-empty spill cells");
21223        let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
21224        let row0 = first.coord.row();
21225        let col0 = first.coord.col();
21226
21227        let mut max_row = row0;
21228        let mut max_col = col0;
21229        let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
21230        for cell in &cells {
21231            max_row = max_row.max(cell.coord.row());
21232            max_col = max_col.max(cell.coord.col());
21233            let v = self
21234                .read_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21235                .unwrap_or(LiteralValue::Empty);
21236            by_coord.insert((cell.coord.row(), cell.coord.col()), v);
21237        }
21238
21239        let rows = (max_row - row0 + 1) as usize;
21240        let cols = (max_col - col0 + 1) as usize;
21241        let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
21242        for r in 0..rows {
21243            let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
21244            for c in 0..cols {
21245                row.push(
21246                    by_coord
21247                        .get(&(row0 + r as u32, col0 + c as u32))
21248                        .cloned()
21249                        .unwrap_or(LiteralValue::Empty),
21250                );
21251            }
21252            values.push(row);
21253        }
21254
21255        Some(SpillSnapshot {
21256            target_cells: cells,
21257            values,
21258        })
21259    }
21260
21261    fn flush_before_range_dependent_vertex(
21262        &mut self,
21263        vertex_id: VertexId,
21264        computed_writes: &mut ComputedWriteBuffer,
21265    ) -> Result<(), ExcelError> {
21266        if self.graph.reads_compressed_range(vertex_id) {
21267            self.flush_computed_write_buffer(computed_writes)?;
21268        }
21269        Ok(())
21270    }
21271
21272    fn plan_vertex_effects_with_computed_flush(
21273        &mut self,
21274        vertex_id: VertexId,
21275        computed_value: LiteralValue,
21276        computed_writes: &mut ComputedWriteBuffer,
21277    ) -> Result<PlannedEffects, ExcelError> {
21278        if matches!(&computed_value, LiteralValue::Array(_)) {
21279            self.flush_computed_write_buffer(computed_writes)?;
21280        }
21281        self.plan_vertex_effects(vertex_id, computed_value)
21282    }
21283
21284    // ── Layer evaluation via effects pipeline ──────────────────────────────
21285
21286    fn evaluate_small_layer_direct_effects(
21287        &mut self,
21288        layer: &super::scheduler::Layer,
21289        delta: Option<&mut DeltaCollector>,
21290        log: Option<&mut ChangeLog>,
21291        cancel_flag: Option<&AtomicBool>,
21292        cancel_check_every: usize,
21293        cancel_message: &'static str,
21294    ) -> Result<usize, ExcelError> {
21295        let cancel = cancel_flag.map(|flag| (flag, cancel_check_every, cancel_message));
21296        self.evaluate_layer_units(layer, delta, log, cancel, false)
21297    }
21298
21299    /// Sequential layer walk over its units (single cells and family runs):
21300    /// each unit evaluates, then its vertices' effects apply in order. With
21301    /// `buffered`, computed writes coalesce in a layer buffer (flushed before
21302    /// a unit that reads a compressed range, before array results, and at
21303    /// the end); otherwise they apply directly. `cancel` = (flag, check every
21304    /// N vertices, message).
21305    fn evaluate_layer_units(
21306        &mut self,
21307        layer: &super::scheduler::Layer,
21308        delta: Option<&mut DeltaCollector>,
21309        log: Option<&mut ChangeLog>,
21310        cancel: Option<(&AtomicBool, usize, &'static str)>,
21311        buffered: bool,
21312    ) -> Result<usize, ExcelError> {
21313        self.evaluate_layer_units_until(layer, delta, log, cancel, buffered, None)
21314    }
21315
21316    /// [`Self::evaluate_layer_units`] that stops before the next unit once
21317    /// `stop_at` has passed; returns the vertices evaluated (a prefix of
21318    /// the layer, all committed).
21319    fn evaluate_layer_units_until(
21320        &mut self,
21321        layer: &super::scheduler::Layer,
21322        mut delta: Option<&mut DeltaCollector>,
21323        mut log: Option<&mut ChangeLog>,
21324        cancel: Option<(&AtomicBool, usize, &'static str)>,
21325        buffered: bool,
21326        stop_at: Option<crate::instant::FzInstant>,
21327    ) -> Result<usize, ExcelError> {
21328        // A chain unit: its run through the chain lift, or else cell by
21329        // cell in row order, each written before the next reads it.
21330        let mut chain_values = None;
21331        if layer.sequential && !layer.runs.is_empty() {
21332            chain_values = match layer.runs.as_slice() {
21333                [run] if run.start == 0 && run.len as usize == layer.vertices.len() => {
21334                    self.try_chain_lift(*run, &layer.vertices)
21335                }
21336                _ => None,
21337            };
21338            if chain_values.is_none() {
21339                let cells = super::scheduler::Layer {
21340                    vertices: layer.vertices.clone(),
21341                    runs: Vec::new(),
21342                    sequential: true,
21343                };
21344                return self.evaluate_layer_units_until(&cells, delta, log, cancel, false, stop_at);
21345            }
21346        }
21347        let chained = chain_values.is_some();
21348        // The chain lift computed every member: one block write, as a run.
21349        let buffered = buffered || chained;
21350        // A dynamic reader's targets are not always ordered before it (its
21351        // pre-probe or observed reads can miss them, e.g. after a structural
21352        // edit). In a buffered layer a member's dirty flag is cleared at its
21353        // commit but its value written at the flush: the members committed
21354        // since the last flush count as dirty for the reader's freshness
21355        // check, which then re-plans it after them.
21356        let track_unflushed = buffered
21357            && self.freshness_armed()
21358            && layer.vertices.iter().any(|&v| self.graph.is_dynamic(v));
21359        let mut committed_unit: &[VertexId] = &[];
21360        let mut computed_writes = ComputedWriteBuffer::default();
21361        let mut next_check = 0usize;
21362        let mut done = 0usize;
21363        for unit in layer_units(layer) {
21364            if done > 0
21365                && let Some(stop_at) = stop_at
21366                && crate::instant::FzInstant::now() >= stop_at
21367            {
21368                self.flush_computed_write_buffer(&mut computed_writes)?;
21369                if track_unflushed {
21370                    self.freshness_flushed();
21371                }
21372                return Ok(done);
21373            }
21374            if let Some((flag, every, message)) = cancel
21375                && every > 0
21376                && done >= next_check
21377            {
21378                next_check = (done / every + 1) * every;
21379                if flag.load(Ordering::Relaxed) {
21380                    if buffered {
21381                        self.flush_computed_write_buffer(&mut computed_writes)?;
21382                    }
21383                    return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21384                        .with_message(message.to_string()));
21385                }
21386            }
21387            if buffered && self.unit_reads_compressed_range(layer, unit) {
21388                self.flush_computed_write_buffer(&mut computed_writes)?;
21389            }
21390            // The previous unit's members are committed (dirty flags
21391            // cleared); while their values wait in the buffer, a dynamic
21392            // reader's read of them is stale (`freshness_dirty_reads`).
21393            if track_unflushed {
21394                if computed_writes.is_empty() {
21395                    self.freshness_flushed();
21396                } else {
21397                    self.freshness_note_unflushed(committed_unit);
21398                }
21399                committed_unit = unit_members(layer, unit);
21400            }
21401            let values = match (chain_values.take(), unit) {
21402                (Some(chain), LayerUnit::Run(run)) => {
21403                    if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21404                        self.flush_computed_write_buffer(&mut computed_writes)?;
21405                        return Err(e);
21406                    }
21407                    let members =
21408                        &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21409                    let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21410                    match self.commit_run_numbers(
21411                        run,
21412                        members,
21413                        &chain,
21414                        delta_active,
21415                        Some(&mut computed_writes),
21416                    ) {
21417                        Ok(true) => {
21418                            done += chain.len();
21419                            continue;
21420                        }
21421                        Ok(false) => {}
21422                        Err(e) => {
21423                            self.flush_computed_write_buffer(&mut computed_writes)?;
21424                            return Err(e);
21425                        }
21426                    }
21427                    members
21428                        .iter()
21429                        .copied()
21430                        .zip(chain.into_iter().map(LiteralValue::Number))
21431                        .collect()
21432                }
21433                (_, unit) => self.evaluate_unit_immutable(layer, unit),
21434            };
21435            // Post-work boundary: the unit ran while (or after) the request
21436            // was cancelled; it is not committed and stays dirty.
21437            if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21438                self.flush_computed_write_buffer(&mut computed_writes)?;
21439                return Err(e);
21440            }
21441            done += values.len();
21442            if let LayerUnit::Run(run) = unit {
21443                let members = &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21444                let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21445                let committed = self.commit_run_scalars(
21446                    run,
21447                    members,
21448                    &values,
21449                    delta_active,
21450                    buffered.then_some(&mut computed_writes),
21451                );
21452                match committed {
21453                    Ok(true) => continue,
21454                    Ok(false) => {}
21455                    Err(e) => {
21456                        self.flush_computed_write_buffer(&mut computed_writes)?;
21457                        return Err(e);
21458                    }
21459                }
21460            }
21461            // A run unit's members were all evaluated before any commits
21462            // (FR5, FORM-192); a single cell is committed as it runs.
21463            let guarded = self.freshness_begin_batch_commit(&values);
21464            for (vertex_id, value) in values {
21465                let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21466                let effects = if buffered {
21467                    self.plan_vertex_effects_with_computed_flush(
21468                        vertex_id,
21469                        value,
21470                        &mut computed_writes,
21471                    )
21472                } else {
21473                    self.plan_vertex_effects(vertex_id, value)
21474                };
21475                let effects = match effects {
21476                    Ok(effects) => effects,
21477                    Err(e) => {
21478                        self.flush_computed_write_buffer(&mut computed_writes)?;
21479                        return Err(e);
21480                    }
21481                };
21482                for effect in &effects {
21483                    if let Err(e) = self.apply_effect_with_computed_writes(
21484                        effect,
21485                        delta.as_deref_mut(),
21486                        log.as_deref_mut(),
21487                        buffered.then_some(&mut computed_writes),
21488                    ) {
21489                        self.flush_computed_write_buffer(&mut computed_writes)?;
21490                        return Err(e);
21491                    }
21492                }
21493                self.freshness_keep_redirtied(redirtied, vertex_id);
21494            }
21495        }
21496        self.flush_computed_write_buffer(&mut computed_writes)?;
21497        if track_unflushed {
21498            self.freshness_flushed();
21499        }
21500        // Debug builds: every chain member equals the per-cell path, now
21501        // that the members above it are written.
21502        #[cfg(debug_assertions)]
21503        if chained {
21504            for &v in &layer.vertices {
21505                let cell = self.graph.get_cell_ref(v);
21506                let (sheet, row, col) = cell
21507                    .map(|c| {
21508                        (
21509                            self.graph.sheet_name(c.sheet_id).to_string(),
21510                            c.coord.row() + 1,
21511                            c.coord.col() + 1,
21512                        )
21513                    })
21514                    .expect("chain member cell");
21515                let written = self.read_cell_value(&sheet, row, col);
21516                let oracle = self
21517                    .evaluate_vertex_immutable(v)
21518                    .unwrap_or_else(LiteralValue::Error);
21519                assert!(
21520                    written
21521                        .as_ref()
21522                        .is_some_and(|w| same_value_bits(w, &oracle)),
21523                    "chain member {sheet}!R{row}C{col}: {written:?} vs per-cell {oracle:?}"
21524                );
21525            }
21526        }
21527        #[cfg(not(debug_assertions))]
21528        let _ = chained;
21529        Ok(layer.vertices.len())
21530    }
21531
21532    /// Evaluate a layer sequentially using the effects pipeline.
21533    fn evaluate_layer_sequential_effects(
21534        &mut self,
21535        layer: &super::scheduler::Layer,
21536    ) -> Result<usize, ExcelError> {
21537        let buffered = buffer_layer_writes(layer);
21538        self.evaluate_layer_units(layer, None, None, None, buffered)
21539    }
21540
21541    /// Evaluate a layer sequentially with delta collection via effects pipeline.
21542    fn evaluate_layer_sequential_with_delta_effects(
21543        &mut self,
21544        layer: &super::scheduler::Layer,
21545        delta: &mut DeltaCollector,
21546    ) -> Result<usize, ExcelError> {
21547        let buffered = buffer_layer_writes(layer);
21548        self.evaluate_layer_units(layer, Some(delta), None, None, buffered)
21549    }
21550
21551    /// Evaluate a layer sequentially with cancellation support via effects pipeline.
21552    fn evaluate_layer_sequential_cancellable_effects(
21553        &mut self,
21554        layer: &super::scheduler::Layer,
21555        cancel_flag: &AtomicBool,
21556    ) -> Result<usize, ExcelError> {
21557        let buffered = buffer_layer_writes(layer);
21558        let cancel = (cancel_flag, 256, "Evaluation cancelled within layer");
21559        self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21560    }
21561
21562    /// Evaluate a layer sequentially with more frequent cancellation for demand-driven eval.
21563    fn evaluate_layer_sequential_cancellable_demand_driven_effects(
21564        &mut self,
21565        layer: &super::scheduler::Layer,
21566        cancel_flag: &AtomicBool,
21567    ) -> Result<usize, ExcelError> {
21568        let buffered = buffer_layer_writes(layer);
21569        let cancel = (
21570            cancel_flag,
21571            128,
21572            "Demand-driven evaluation cancelled within layer",
21573        );
21574        self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21575    }
21576
21577    /// Evaluate a layer in parallel, applying via effects pipeline.
21578    fn evaluate_layer_parallel_effects(
21579        &mut self,
21580        layer: &super::scheduler::Layer,
21581        min_chunk: u32,
21582    ) -> Result<usize, ExcelError> {
21583        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21584
21585        let phases = self.parallel_phases(layer);
21586        let mut applied = 0usize;
21587
21588        for (units, group) in &phases {
21589            let group = &group[..];
21590            if group.is_empty() {
21591                continue;
21592            }
21593            let mut computed_writes = ComputedWriteBuffer::default();
21594
21595            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21596                thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, min_chunk));
21597
21598            // Post-work boundary: a group evaluated across a live
21599            // cancellation is not committed (it stays dirty).
21600            let results = results.and_then(|results| {
21601                self.live_cancellation_after_work(GROUP_CANCELLED)
21602                    .map(|()| results)
21603            });
21604            // FR3: a parallel group is one commit unit; one stale reader
21605            // drops the whole group (it stays dirty and replans).
21606            self.freshness_gate_group(group);
21607            match results {
21608                Ok(vertex_results) => {
21609                    let (vertex_results, committed) = self.commit_parallel_runs(
21610                        layer,
21611                        units,
21612                        vertex_results,
21613                        false,
21614                        &mut computed_writes,
21615                    )?;
21616                    applied = applied.saturating_add(committed);
21617                    // Arrays first, then scalars — establishes spill regions before
21618                    // scalar results that might land inside a spilled region.
21619                    // FR5 (FORM-192): members evaluated together must not
21620                    // clear a re-dirty from a spill committed before them.
21621                    let guarded = self.freshness_begin_batch_commit(&vertex_results);
21622                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21623                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21624                    for (vertex_id, result) in vertex_results {
21625                        if matches!(result, LiteralValue::Array(_)) {
21626                            arrays.push((vertex_id, result));
21627                        } else {
21628                            others.push((vertex_id, result));
21629                        }
21630                    }
21631                    for (vertex_id, result) in arrays {
21632                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21633                        let effects = match self.plan_vertex_effects_with_computed_flush(
21634                            vertex_id,
21635                            result,
21636                            &mut computed_writes,
21637                        ) {
21638                            Ok(effects) => effects,
21639                            Err(e) => {
21640                                self.flush_computed_write_buffer(&mut computed_writes)?;
21641                                return Err(e);
21642                            }
21643                        };
21644                        for effect in &effects {
21645                            if let Err(e) = self.apply_effect_with_computed_writes(
21646                                effect,
21647                                None,
21648                                None,
21649                                Some(&mut computed_writes),
21650                            ) {
21651                                self.flush_computed_write_buffer(&mut computed_writes)?;
21652                                return Err(e);
21653                            }
21654                        }
21655                        self.freshness_keep_redirtied(redirtied, vertex_id);
21656                        applied = applied.saturating_add(1);
21657                    }
21658                    // Make all array spill/top-left writes visible before scalar effects in this group.
21659                    self.flush_computed_write_buffer(&mut computed_writes)?;
21660                    for (vertex_id, result) in others {
21661                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21662                        let effects = match self.plan_vertex_effects_with_computed_flush(
21663                            vertex_id,
21664                            result,
21665                            &mut computed_writes,
21666                        ) {
21667                            Ok(effects) => effects,
21668                            Err(e) => {
21669                                self.flush_computed_write_buffer(&mut computed_writes)?;
21670                                return Err(e);
21671                            }
21672                        };
21673                        for effect in &effects {
21674                            if let Err(e) = self.apply_effect_with_computed_writes(
21675                                effect,
21676                                None,
21677                                None,
21678                                Some(&mut computed_writes),
21679                            ) {
21680                                self.flush_computed_write_buffer(&mut computed_writes)?;
21681                                return Err(e);
21682                            }
21683                        }
21684                        self.freshness_keep_redirtied(redirtied, vertex_id);
21685                        applied = applied.saturating_add(1);
21686                    }
21687                    // Flush at the group boundary; phase1 must be visible before phase2.
21688                    self.flush_computed_write_buffer(&mut computed_writes)?;
21689                }
21690                Err(e) => {
21691                    self.flush_computed_write_buffer(&mut computed_writes)?;
21692                    return Err(e);
21693                }
21694            }
21695        }
21696
21697        Ok(applied)
21698    }
21699
21700    /// Evaluate a layer in parallel with delta collection via effects pipeline.
21701    fn evaluate_layer_parallel_with_delta_effects(
21702        &mut self,
21703        layer: &super::scheduler::Layer,
21704        delta: &mut DeltaCollector,
21705    ) -> Result<usize, ExcelError> {
21706        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21707
21708        let phases = self.parallel_phases(layer);
21709        let mut applied = 0usize;
21710
21711        for (units, group) in &phases {
21712            let group = &group[..];
21713            if group.is_empty() {
21714                continue;
21715            }
21716            let mut computed_writes = ComputedWriteBuffer::default();
21717            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21718                thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, 8));
21719
21720            // Post-work boundary: a group evaluated across a live
21721            // cancellation is not committed (it stays dirty).
21722            let results = results.and_then(|results| {
21723                self.live_cancellation_after_work(GROUP_CANCELLED)
21724                    .map(|()| results)
21725            });
21726            // FR3: a parallel group is one commit unit; one stale reader
21727            // drops the whole group (it stays dirty and replans).
21728            self.freshness_gate_group(group);
21729            match results {
21730                Ok(vertex_results) => {
21731                    let (vertex_results, committed) = self.commit_parallel_runs(
21732                        layer,
21733                        units,
21734                        vertex_results,
21735                        delta.mode != DeltaMode::Off,
21736                        &mut computed_writes,
21737                    )?;
21738                    applied = applied.saturating_add(committed);
21739                    // FR5 (FORM-192): members evaluated together must not
21740                    // clear a re-dirty from a spill committed before them.
21741                    let guarded = self.freshness_begin_batch_commit(&vertex_results);
21742                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21743                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21744                    for (vertex_id, result) in vertex_results {
21745                        if matches!(result, LiteralValue::Array(_)) {
21746                            arrays.push((vertex_id, result));
21747                        } else {
21748                            others.push((vertex_id, result));
21749                        }
21750                    }
21751                    for (vertex_id, result) in arrays {
21752                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21753                        let effects = match self.plan_vertex_effects_with_computed_flush(
21754                            vertex_id,
21755                            result,
21756                            &mut computed_writes,
21757                        ) {
21758                            Ok(effects) => effects,
21759                            Err(e) => {
21760                                self.flush_computed_write_buffer(&mut computed_writes)?;
21761                                return Err(e);
21762                            }
21763                        };
21764                        for effect in &effects {
21765                            if let Err(e) = self.apply_effect_with_computed_writes(
21766                                effect,
21767                                Some(delta),
21768                                None,
21769                                Some(&mut computed_writes),
21770                            ) {
21771                                self.flush_computed_write_buffer(&mut computed_writes)?;
21772                                return Err(e);
21773                            }
21774                        }
21775                        self.freshness_keep_redirtied(redirtied, vertex_id);
21776                        applied = applied.saturating_add(1);
21777                    }
21778                    self.flush_computed_write_buffer(&mut computed_writes)?;
21779                    for (vertex_id, result) in others {
21780                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21781                        let effects = match self.plan_vertex_effects_with_computed_flush(
21782                            vertex_id,
21783                            result,
21784                            &mut computed_writes,
21785                        ) {
21786                            Ok(effects) => effects,
21787                            Err(e) => {
21788                                self.flush_computed_write_buffer(&mut computed_writes)?;
21789                                return Err(e);
21790                            }
21791                        };
21792                        for effect in &effects {
21793                            if let Err(e) = self.apply_effect_with_computed_writes(
21794                                effect,
21795                                Some(delta),
21796                                None,
21797                                Some(&mut computed_writes),
21798                            ) {
21799                                self.flush_computed_write_buffer(&mut computed_writes)?;
21800                                return Err(e);
21801                            }
21802                        }
21803                        self.freshness_keep_redirtied(redirtied, vertex_id);
21804                        applied = applied.saturating_add(1);
21805                    }
21806                    self.flush_computed_write_buffer(&mut computed_writes)?;
21807                }
21808                Err(e) => {
21809                    self.flush_computed_write_buffer(&mut computed_writes)?;
21810                    return Err(e);
21811                }
21812            }
21813        }
21814
21815        Ok(applied)
21816    }
21817
21818    /// Evaluate a layer in parallel with cancellation support via effects pipeline.
21819    fn evaluate_layer_parallel_cancellable_effects(
21820        &mut self,
21821        layer: &super::scheduler::Layer,
21822        cancel_flag: &AtomicBool,
21823    ) -> Result<usize, ExcelError> {
21824        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21825
21826        if cancel_flag.load(Ordering::Relaxed) {
21827            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21828                .with_message("Parallel evaluation cancelled before starting".to_string()));
21829        }
21830
21831        let phases = self.parallel_phases(layer);
21832        let mut applied = 0usize;
21833
21834        for (units, group) in &phases {
21835            let group = &group[..];
21836            if group.is_empty() {
21837                continue;
21838            }
21839            let mut computed_writes = ComputedWriteBuffer::default();
21840
21841            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> = thread_pool
21842                .install(|| self.evaluate_units_parallel(layer, units, Some(cancel_flag), 8));
21843
21844            // Post-work boundary: a group evaluated across a live
21845            // cancellation is not committed (it stays dirty).
21846            let results = results.and_then(|results| {
21847                self.live_cancellation_after_work(GROUP_CANCELLED)
21848                    .map(|()| results)
21849            });
21850            // FR3: a parallel group is one commit unit; one stale reader
21851            // drops the whole group (it stays dirty and replans).
21852            self.freshness_gate_group(group);
21853            match results {
21854                Ok(vertex_results) => {
21855                    let (vertex_results, committed) = self.commit_parallel_runs(
21856                        layer,
21857                        units,
21858                        vertex_results,
21859                        false,
21860                        &mut computed_writes,
21861                    )?;
21862                    applied = applied.saturating_add(committed);
21863                    // FR5 (FORM-192): members evaluated together must not
21864                    // clear a re-dirty from a spill committed before them.
21865                    let guarded = self.freshness_begin_batch_commit(&vertex_results);
21866                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21867                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21868                    for (vertex_id, result) in vertex_results {
21869                        if matches!(result, LiteralValue::Array(_)) {
21870                            arrays.push((vertex_id, result));
21871                        } else {
21872                            others.push((vertex_id, result));
21873                        }
21874                    }
21875                    for (vertex_id, result) in arrays {
21876                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21877                        let effects = match self.plan_vertex_effects_with_computed_flush(
21878                            vertex_id,
21879                            result,
21880                            &mut computed_writes,
21881                        ) {
21882                            Ok(effects) => effects,
21883                            Err(e) => {
21884                                self.flush_computed_write_buffer(&mut computed_writes)?;
21885                                return Err(e);
21886                            }
21887                        };
21888                        for effect in &effects {
21889                            if let Err(e) = self.apply_effect_with_computed_writes(
21890                                effect,
21891                                None,
21892                                None,
21893                                Some(&mut computed_writes),
21894                            ) {
21895                                self.flush_computed_write_buffer(&mut computed_writes)?;
21896                                return Err(e);
21897                            }
21898                        }
21899                        self.freshness_keep_redirtied(redirtied, vertex_id);
21900                        applied = applied.saturating_add(1);
21901                    }
21902                    self.flush_computed_write_buffer(&mut computed_writes)?;
21903                    for (vertex_id, result) in others {
21904                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21905                        let effects = match self.plan_vertex_effects_with_computed_flush(
21906                            vertex_id,
21907                            result,
21908                            &mut computed_writes,
21909                        ) {
21910                            Ok(effects) => effects,
21911                            Err(e) => {
21912                                self.flush_computed_write_buffer(&mut computed_writes)?;
21913                                return Err(e);
21914                            }
21915                        };
21916                        for effect in &effects {
21917                            if let Err(e) = self.apply_effect_with_computed_writes(
21918                                effect,
21919                                None,
21920                                None,
21921                                Some(&mut computed_writes),
21922                            ) {
21923                                self.flush_computed_write_buffer(&mut computed_writes)?;
21924                                return Err(e);
21925                            }
21926                        }
21927                        self.freshness_keep_redirtied(redirtied, vertex_id);
21928                        applied = applied.saturating_add(1);
21929                    }
21930                    self.flush_computed_write_buffer(&mut computed_writes)?;
21931                }
21932                Err(e) => {
21933                    self.flush_computed_write_buffer(&mut computed_writes)?;
21934                    return Err(e);
21935                }
21936            }
21937        }
21938
21939        Ok(applied)
21940    }
21941
21942    // ── Top-level evaluate_all_logged ───────────────────────────────────────
21943
21944    /// Evaluate all dirty/volatile vertices, recording effects into a ChangeLog.
21945    ///
21946    /// This is the same flow as `evaluate_all` but threads a ChangeLog through
21947    /// every effect application so that spill commits/clears are captured.
21948    pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
21949        self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
21950            engine.evaluate_all_logged_unobserved(log)
21951        })
21952    }
21953
21954    fn evaluate_all_logged_unobserved(
21955        &mut self,
21956        log: &mut ChangeLog,
21957    ) -> Result<EvalResult, ExcelError> {
21958        self.observe_function_semantic_epoch()?;
21959        let _source_cache = self.source_cache_session();
21960        self.validate_deterministic_mode()?;
21961        if self.config.defer_graph_building {
21962            self.build_graph_all()?;
21963        }
21964        self.require_unified_authority()?;
21965        self.begin_evaluation_request();
21966        self.reset_virtual_dep_telemetry_if_disabled();
21967        let start = crate::instant::FzInstant::now();
21968        let mut computed_vertices = 0;
21969        let mut cycle_errors = 0;
21970
21971        let mut replan_iterations = 0;
21972        const MAX_REPLAN: usize = 5;
21973        let mut telemetry = self
21974            .config
21975            .enable_virtual_dep_telemetry
21976            .then(|| self.start_virtual_dep_telemetry());
21977
21978        log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
21979
21980        let result = (|| -> Result<EvalResult, ExcelError> {
21981            loop {
21982                let to_evaluate = self.graph.get_evaluation_vertices();
21983                if to_evaluate.is_empty() {
21984                    if let Some(t) = telemetry.as_mut()
21985                        && t.bailout_reason.is_none()
21986                    {
21987                        t.bailout_reason = Some("no_work");
21988                    }
21989                    break;
21990                }
21991
21992                let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21993                if let Some(t) = telemetry.as_mut() {
21994                    Self::accumulate_schedule_meta(t, &meta);
21995                }
21996
21997                // Walk units in condensation order: stamp cycles at their
21998                // position, evaluate layers with ChangeLog recording.
21999                self.begin_pass(&schedule);
22000                for (unit_index, &unit) in schedule.units.iter().enumerate() {
22001                    match unit {
22002                        ScheduleUnit::Cycle(i) => {
22003                            // Journal integration (design doc §4 last row): the
22004                            // ChangeLog in this path only records SpillClear /
22005                            // SpillCommit events; WriteCell effects are never
22006                            // logged (see `apply_write_cell`). Runtime SCC tasks
22007                            // write values directly and never spill (§7.9 stamps
22008                            // would-be anchors), and their spill *teardown* is the
22009                            // same unlogged `stamp_cycle_error` the Static path
22010                            // already uses here — so direct commits coexist with
22011                            // the journal cleanly, with identical semantics to
22012                            // Static. Pinned by `scc_runtime_cycles` tests.
22013                            if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
22014                            {
22015                                cycle_errors += 1;
22016                            }
22017                        }
22018                        ScheduleUnit::Layer(i) => {
22019                            computed_vertices +=
22020                                self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
22021                        }
22022                    }
22023                    if self.stop_after_unit(&schedule, unit_index) {
22024                        break;
22025                    }
22026                }
22027
22028                let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
22029                if let Some(t) = telemetry.as_mut() {
22030                    t.changed_vdeps_total += changed_vertices.len();
22031                }
22032                self.resource_checkpoint(0)?;
22033                if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
22034                    if let Some(t) = telemetry.as_mut() {
22035                        t.bailout_reason = Some("converged");
22036                    }
22037                    break;
22038                }
22039                if replan_iterations >= MAX_REPLAN {
22040                    if let Some(mut t) = telemetry.take() {
22041                        t.bailout_reason = Some("max_replan");
22042                        t.replan_iterations = replan_iterations;
22043                        self.last_virtual_dep_telemetry = t;
22044                    }
22045                    return Err(
22046                        self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
22047                    );
22048                }
22049                replan_iterations += 1;
22050            }
22051
22052            if let Some(mut t) = telemetry {
22053                t.replan_iterations = replan_iterations;
22054                self.last_virtual_dep_telemetry = t;
22055            }
22056
22057            self.redirty_for_next_recalc();
22058            self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
22059
22060            Ok(EvalResult {
22061                computed_vertices,
22062                cycle_errors,
22063                elapsed: start.elapsed(),
22064            })
22065        })();
22066        log.end_compound();
22067        result
22068    }
22069
22070    /// Evaluate a single layer with ChangeLog recording.
22071    fn evaluate_layer_logged(
22072        &mut self,
22073        layer: &super::scheduler::Layer,
22074        log: &mut ChangeLog,
22075    ) -> Result<usize, ExcelError> {
22076        self.resource_checkpoint(layer.vertices.len() as u64)?;
22077        self.evaluate_layer_units(layer, None, Some(log), None, true)
22078    }
22079}