Skip to main content

formualizer_eval/engine/
eval.rs

1use crate::SheetId;
2
3mod criteria;
4mod exact;
5mod family;
6mod kernels;
7mod lift;
8mod memo;
9use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
10#[cfg(test)]
11use crate::engine::Scheduler;
12use crate::engine::arena::AstNodeId;
13use crate::engine::eval_delta::{
14    DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
15};
16use crate::engine::graph::editor::change_log::MutationCapture;
17use crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan;
18use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
19use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
20use crate::engine::live_graph::analyze_live_graph;
21use crate::engine::lookup_index_cache::{
22    BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
23    LookupIndexKey, estimate_bytes,
24};
25use crate::engine::named_range::{NameScope, NamedDefinition};
26use crate::engine::range_view::RangeView;
27use crate::engine::row_visibility::RowVisibilityState;
28use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
29use crate::engine::target_preparation::{
30    StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
31};
32use crate::engine::used_extent::{
33    ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
34};
35use crate::engine::virtual_deps::VirtualDepBuilder;
36use family::{LayerUnit, layer_units, unit_members};
37
38#[path = "freshness.rs"]
39mod freshness;
40use crate::engine::template::region::Region;
41use crate::engine::{
42    ChangeLogger, CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
43    EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceRequestStats,
44    FormulaDirtyLeaseOutcome, FormulaIngestBatch, FormulaIngestRecord, FormulaIngestReport,
45    FormulaParseDiagnostic, FormulaParsePolicy, FormulaPlaneMode, ResourceLedger,
46    RowVisibilitySource, ScheduleUnit, VertexId, VertexKind, VisibilityMaskMode,
47};
48use crate::function::FnCaps;
49use crate::interpreter::Interpreter;
50use crate::reference::{CellRef, Coord, RangeRef};
51use crate::traits::FunctionProvider;
52use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
53use formualizer_common::{
54    CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
55};
56use formualizer_parse::parser::ReferenceType;
57use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
58use rayon::ThreadPoolBuilder;
59use rustc_hash::{FxHashMap, FxHashSet};
60use std::collections::{BTreeMap, BTreeSet, VecDeque};
61use std::sync::Arc;
62use std::sync::atomic::{AtomicBool, Ordering};
63use std::time::Instant;
64
65type StagedFormulaEntry = (u32, u32, String);
66type StagedSheetParts = (
67    Vec<StagedFormulaEntry>,
68    Option<crate::engine::DeferredFormulaPackage>,
69);
70
71/// Per-sheet staged-formula store (NOTE(#126) follow-up).
72///
73/// Ingest consumers (`build_graph_all`/`build_graph_for_sheets`) walk staged
74/// entries in INSERTION order, so the order-preserving `Vec` stays the
75/// canonical storage; a `(row, col) → index` map removes the linear dup-scan
76/// that made `stage_formula_text`/`get_staged_formula_text` O(staged-on-sheet)
77/// per call (O(n²) for an n-formula deferred load on one sheet — ~570 ms for
78/// 50k stages, release, before the index). `stage`/`get` are O(1);
79/// `remove` keeps the old O(n) `Vec::remove` (rare path, order preserved).
80#[derive(Default)]
81pub(crate) struct StagedSheet {
82    entries: Vec<StagedFormulaEntry>,
83    index: FxHashMap<(u32, u32), usize>,
84    deferred_package: Option<crate::engine::DeferredFormulaPackage>,
85}
86
87impl StagedSheet {
88    fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
89        package
90            .invalidated
91            .extend(package.families.iter().map(|family| family.source_id));
92        package.invalidated.extend(
93            package
94                .partitioned_families
95                .iter()
96                .map(|family| family.source_id),
97        );
98    }
99
100    fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
101        let Some(package) = self.deferred_package.as_mut() else {
102            return;
103        };
104        if package.suppressed.contains(&(row, col)) {
105            // Previously consumed coordinates are already excluded from residual
106            // authority. Edits/deletion must not invalidate untouched fragments.
107            return;
108        }
109        if package.source_geometry_complete
110            && package.families.is_empty()
111            && package.partitioned_families.is_empty()
112        {
113            if let Some((row0, col0)) = row.checked_sub(1).zip(col.checked_sub(1))
114                && package
115                    .source_coordinates
116                    .binary_search(&crate::engine::SourceCoord {
117                        row: row0,
118                        col: col0,
119                    })
120                    .is_ok()
121            {
122                package.suppressed.insert((row, col));
123            }
124            return;
125        }
126        let family = package
127            .replay
128            .lock()
129            .ok()
130            .and_then(|mut replay| replay.formula_at(row, col).ok())
131            .flatten()
132            .and_then(|record| record.partition_owner.or(record.family));
133        if let Some(family) = family {
134            package.invalidated.insert(family);
135        } else {
136            // A poisoned lock, replay failure, malformed spool, or missing
137            // lookup result must never let a possibly edited family commit.
138            // Fail closed at package scope before suppressing the edited cell.
139            Self::invalidate_all_deferred_families(package);
140        }
141        package.suppressed.insert((row, col));
142    }
143
144    fn reconcile_attached_deferred_package(&mut self) {
145        let Some(package) = self.deferred_package.as_mut() else {
146            return;
147        };
148        if self.entries.is_empty() {
149            return;
150        }
151
152        let replayed = (|| {
153            let mut disposition = crate::engine::FormulaReplayDisposition::default();
154            for partition in &package.partitioned_families {
155                disposition.register_partition(partition, false)?;
156            }
157            package
158                .replay
159                .lock()
160                .map_err(|_| "deferred formula spool lock poisoned".to_string())?
161                .replay_partitioned(&disposition, &package.partitioned_families)
162        })();
163
164        let Ok(records) = replayed else {
165            Self::invalidate_all_deferred_families(package);
166            package
167                .suppressed
168                .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
169            return;
170        };
171        let mut owners = FxHashMap::default();
172        for record in &records {
173            owners
174                .entry((record.row, record.col))
175                .or_insert(record.partition_owner.or(record.family));
176        }
177        for (row, col, _) in &self.entries {
178            if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
179                package.invalidated.insert(family);
180            } else {
181                Self::invalidate_all_deferred_families(package);
182            }
183            package.suppressed.insert((*row, *col));
184        }
185        package.reconciliation_replay = Some(records);
186    }
187
188    fn stage(&mut self, row: u32, col: u32, text: String) {
189        self.invalidate_deferred_at(row, col);
190        match self.index.entry((row, col)) {
191            std::collections::hash_map::Entry::Occupied(slot) => {
192                self.entries[*slot.get()].2 = text;
193            }
194            std::collections::hash_map::Entry::Vacant(slot) => {
195                slot.insert(self.entries.len());
196                self.entries.push((row, col, text));
197            }
198        }
199    }
200
201    fn remove(&mut self, row: u32, col: u32) -> Option<String> {
202        let deferred_text = self.get(row, col);
203        self.invalidate_deferred_at(row, col);
204        let Some(idx) = self.index.remove(&(row, col)) else {
205            return deferred_text;
206        };
207        let (_, _, text) = self.entries.remove(idx);
208        // `Vec::remove` shifted everything after `idx` left by one.
209        for slot in self.index.values_mut() {
210            if *slot > idx {
211                *slot -= 1;
212            }
213        }
214        Some(text)
215    }
216
217    fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
218        self.index
219            .get(&(row, col))
220            .map(|&i| self.entries[i].2.as_str())
221    }
222
223    fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
224        let idx = self.index.remove(&(row, col))?;
225        let (_, _, text) = self.entries.remove(idx);
226        for slot in self.index.values_mut() {
227            if *slot > idx {
228                *slot -= 1;
229            }
230        }
231        Some(text)
232    }
233
234    fn get(&self, row: u32, col: u32) -> Option<String> {
235        if let Some(text) = self.get_ordinary(row, col) {
236            return Some(text.to_string());
237        }
238        let package = self.deferred_package.as_ref()?;
239        if package.suppressed.contains(&(row, col)) {
240            return None;
241        }
242        package
243            .replay
244            .lock()
245            .ok()?
246            .formula_at(row, col)
247            .ok()?
248            .map(|record| record.text)
249    }
250
251    fn len(&self) -> usize {
252        self.deferred_package
253            .as_ref()
254            .map_or(self.entries.len(), |package| {
255                (if package.source_geometry_complete
256                    && (package.coordinates_cover_families
257                        || (package.families.is_empty() && package.partitioned_families.is_empty()))
258                {
259                    package.source_coordinates.len()
260                } else {
261                    usize::try_from(package.report.source_formula_records_spooled)
262                        .unwrap_or(usize::MAX)
263                })
264                .saturating_sub(package.suppressed.len())
265                .saturating_add(self.entries.len())
266            })
267    }
268
269    fn is_empty(&self) -> bool {
270        self.entries.is_empty() && self.deferred_package.is_none()
271    }
272
273    fn into_parts(self) -> StagedSheetParts {
274        (self.entries, self.deferred_package)
275    }
276}
277
278type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
279
280#[derive(Clone, Debug, PartialEq, Eq, Hash)]
281struct PreparationRegion {
282    sheet: String,
283    sheet_id: SheetId,
284    start_row: u32,
285    start_col: u32,
286    end_row: u32,
287    end_col: u32,
288}
289
290// Target roots dedupe by identity; they keep their first-seen order.
291struct OrderedTargetProducers {
292    ordered: Vec<crate::engine::target_preparation::TargetProducer>,
293    seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
294}
295
296impl OrderedTargetProducers {
297    fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
298        let mut ordered = Vec::new();
299        ordered.try_reserve(capacity)?;
300        let mut seen = FxHashSet::default();
301        seen.try_reserve(capacity)?;
302        Ok(Self { ordered, seen })
303    }
304
305    fn from_ordered(
306        ordered: Vec<crate::engine::target_preparation::TargetProducer>,
307    ) -> Result<Self, std::collections::TryReserveError> {
308        let mut seen = FxHashSet::default();
309        seen.try_reserve(ordered.len())?;
310        seen.extend(ordered.iter().copied());
311        Ok(Self { ordered, seen })
312    }
313
314    fn push(
315        &mut self,
316        producer: crate::engine::target_preparation::TargetProducer,
317    ) -> Result<bool, std::collections::TryReserveError> {
318        #[cfg(test)]
319        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
320        if self.seen.contains(&producer) {
321            return Ok(false);
322        }
323        self.seen.try_reserve(1)?;
324        self.ordered.try_reserve(1)?;
325        self.seen.insert(producer);
326        self.ordered.push(producer);
327        Ok(true)
328    }
329
330    fn len(&self) -> usize {
331        self.ordered.len()
332    }
333
334    fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
335        self.ordered
336    }
337}
338
339#[cfg(test)]
340thread_local! {
341    static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
342}
343
344fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
345    crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
346        reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
347        limit: u64::MAX,
348        observed: observed as u64,
349        request_id,
350    })
351    .into_excel_error()
352}
353
354#[derive(Debug)]
355struct PreparedOrdinaryStagedFormula {
356    sheet: String,
357    sheet_id: SheetId,
358    lease: StagedFormulaLease,
359    ast_id: Option<AstNodeId>,
360    plan: Option<DependencyPlanRow>,
361}
362
363struct PreparedTargetSourcePackage {
364    sheet: String,
365    sheet_id: SheetId,
366    lease: StagedPackageLease,
367    selected_points: Option<BTreeSet<(u32, u32)>>,
368    complete_selections: BTreeSet<crate::engine::SourceFamilyId>,
369    deferred_shared: bool,
370    direct_domains: Vec<(
371        crate::engine::SourceFamilyId,
372        crate::engine::PlacementDomainTransport,
373    )>,
374    source_report: crate::engine::FormulaCompressedSourceReport,
375    replay_records: Vec<crate::engine::DeferredReplayFormula>,
376    spool_replays: u64,
377    disposition: crate::engine::FormulaReplayDisposition,
378    legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
379    direct_families: usize,
380    direct_cells: u64,
381    direct_fragments: u64,
382    direct_complete_families: u64,
383    direct_complete_cells: u64,
384    direct_partition_families: u64,
385    direct_partition_cells: u64,
386    anchor_parses: u64,
387    anchor_asts: u64,
388    anchor_analyses: u64,
389}
390
391impl PreparedTargetSourcePackage {
392    fn empty_selection(sheet: &str, sheet_id: SheetId, lease: StagedPackageLease) -> Self {
393        Self {
394            sheet: sheet.to_owned(),
395            sheet_id,
396            lease,
397            selected_points: Some(BTreeSet::new()),
398            complete_selections: Default::default(),
399            deferred_shared: false,
400            direct_domains: Vec::new(),
401            source_report: Default::default(),
402            replay_records: Vec::new(),
403            spool_replays: 0,
404            disposition: Default::default(),
405            legacy: Vec::new(),
406            direct_families: 0,
407            direct_cells: 0,
408            direct_fragments: 0,
409            direct_complete_families: 0,
410            direct_complete_cells: 0,
411            direct_partition_families: 0,
412            direct_partition_cells: 0,
413            anchor_parses: 0,
414            anchor_asts: 0,
415            anchor_analyses: 0,
416        }
417    }
418
419    fn direct_contains(&self, row: u32, col: u32) -> bool {
420        self.direct_domains.iter().any(|(_, domain)| {
421            let rect = domain.rect();
422            row > rect.start.row
423                && row <= rect.end.row + 1
424                && col > rect.start.col
425                && col <= rect.end.col + 1
426        })
427    }
428
429    fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
430        self.replay_records.iter().filter(|record| {
431            let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
432                return true;
433            };
434            let coord = crate::engine::SourceCoord { row, col };
435            let disposition = match record.family {
436                Some(family) => self.disposition.shared_disposition(family, coord),
437                None => self.disposition.ordinary_disposition(coord).0,
438            };
439            !matches!(
440                disposition,
441                crate::engine::FormulaReplayCoordinateDisposition::Direct
442                    | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
443            )
444        })
445    }
446}
447
448type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
449type StagedFormulaBatches = Vec<(String, StagedSheet)>;
450
451type CompressedReplayBatch = (
452    FormulaIngestBatch,
453    crate::engine::FormulaCompressedSourceBatch,
454);
455type PreparedSourceBatch = (
456    FormulaIngestBatch,
457    crate::engine::FormulaCompressedSourceReport,
458    crate::engine::FormulaCompressedPreparation,
459);
460type PreparedStagedFormulaBatches = (
461    PreparedFormulaBatches,
462    Vec<CompressedReplayBatch>,
463    Vec<PreparedSourceBatch>,
464    // Formulas whose planning can fail (see `formula_may_fail_planning`).
465    FxHashSet<crate::engine::arena::AstNodeId>,
466);
467
468/// Backend-neutral source-family ingress. Adapters may prepare candidates and
469/// submit exact replay; every formula is still materialized per cell.
470#[doc(hidden)]
471pub struct SourceFormulaIngress<'a, R> {
472    engine: &'a mut Engine<R>,
473}
474
475impl<R> SourceFormulaIngress<'_, R>
476where
477    R: EvaluationContext,
478{
479    pub fn prepare_families(
480        &mut self,
481        sheet_name: &str,
482        families: &[crate::engine::SourceFormulaFamily],
483    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
484        self.engine.observe_function_semantic_epoch()?;
485        Ok(self
486            .engine
487            .prepare_source_formula_families(sheet_name, families))
488    }
489
490    pub fn prepare_eager_proposals(
491        &mut self,
492        sheet_name: &str,
493        families: &[crate::engine::SourceFormulaFamily],
494        partitions: &[crate::engine::PartitionedSourceFormulaFamily],
495        formula_record_count: u64,
496        replay: Box<dyn crate::engine::DeferredFormulaReplay>,
497    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
498        self.engine.observe_function_semantic_epoch()?;
499        let replay = Arc::new(std::sync::Mutex::new(replay));
500        self.engine.prepare_source_formula_proposals(
501            sheet_name,
502            families,
503            partitions,
504            partitions,
505            formula_record_count,
506            replay,
507            &BTreeSet::new(),
508            &Default::default(),
509            None,
510        )
511    }
512
513    pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
514        self.engine.stage_deferred_formula_package(package);
515    }
516
517    pub fn ingest_replay_batches(
518        &mut self,
519        batches: Vec<(
520            FormulaIngestBatch,
521            crate::engine::FormulaCompressedSourceBatch,
522        )>,
523    ) -> Result<FormulaIngestReport, ExcelError> {
524        self.engine
525            .ingest_compressed_formula_source_batches(batches)
526    }
527
528    pub fn finish_prepared(
529        &mut self,
530        batches: Vec<(
531            FormulaIngestBatch,
532            crate::engine::FormulaCompressedSourceReport,
533            crate::engine::FormulaCompressedPreparation,
534        )>,
535    ) -> Result<FormulaIngestReport, ExcelError> {
536        self.engine.finish_compressed_formula_sources(batches)
537    }
538}
539
540// Computed-write coalescing pays a fixed grouping/planning cost. For very narrow
541// layers there is not enough work to amortize it, and the direct point-write path
542// is faster while preserving the same visibility semantics.
543const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
544
545/// Equal values, numbers by bits (debug oracles).
546#[cfg(debug_assertions)]
547fn same_value_bits(a: &LiteralValue, b: &LiteralValue) -> bool {
548    match (a, b) {
549        (LiteralValue::Number(x), LiteralValue::Number(y)) => x.to_bits() == y.to_bits(),
550        _ => a == b,
551    }
552}
553
554/// Whether a layer's computed writes are buffered: not for a chain unit,
555/// whose members read the ones written before them.
556fn buffer_layer_writes(layer: &crate::engine::scheduler::Layer) -> bool {
557    !layer.sequential && layer.vertices.len() >= COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH
558}
559/// Adaptive layer parallelism (see `evaluate_layer_parallel`): a layer runs
560/// sequentially until either it has run `PARALLEL_LAYER_PROBE` or the rest,
561/// at the rate so far, is estimated at `PARALLEL_LAYER_WORTH` or more; then
562/// the rest goes to the thread pool. Tuned on Enron first eval (probe 30 /
563/// 100 / 300 µs, worth 50 / 150 / 400 µs; 300 / 150 best, no workbook
564/// slower).
565/// Candidate count from which schedule preparation sorts on the pool.
566const PARALLEL_SCHEDULE_MIN_CANDIDATES: usize = 16 * 1024;
567/// Plan reuse: a schedule of more candidates than this can become the
568/// base, and a request takes the base restricted to it when the base is at
569/// most `BASE_SCHEDULE_RATIO` times the request.
570const BASE_SCHEDULE_MIN_VERTICES: usize = 64;
571const BASE_SCHEDULE_RATIO: usize = 256;
572/// A restricted schedule keeps the base's layers, which can split a family
573/// run the planner would keep whole (each piece then builds its own
574/// criteria index): below this many candidates planning costs less than
575/// that (real_ops_model: 75 candidates, 31 us to plan, +0.3 ms restricted).
576const BASE_SCHEDULE_MIN_REQUEST: usize = 128;
577const PARALLEL_LAYER_PROBE: std::time::Duration = std::time::Duration::from_micros(300);
578const PARALLEL_LAYER_WORTH: std::time::Duration = std::time::Duration::from_micros(150);
579/// A member this expensive (ns, measured by the probe) is its own task.
580const EXPENSIVE_VERTEX_NS: u128 = 10_000;
581/// [`Engine::live_cancellation_after_work`] messages.
582const UNIT_CANCELLED: &str = "Evaluation cancelled; the evaluated unit was not committed";
583const GROUP_CANCELLED: &str =
584    "Parallel evaluation cancelled; the evaluated group was not committed";
585const SCC_CANCELLED: &str = "Evaluation cancelled; the evaluated cycle was not completed";
586
587#[derive(Debug, Clone, PartialEq)]
588pub(crate) enum ComputedWrite {
589    Cell {
590        seq: u64,
591        sheet_id: SheetId,
592        row0: u32,
593        col0: u32,
594        value: OverlayValue,
595        format_id: Option<crate::format::FormatId>,
596    },
597    Rect {
598        seq: u64,
599        sheet_id: SheetId,
600        sr0: u32,
601        sc0: u32,
602        values: Vec<Vec<OverlayValue>>,
603    },
604    /// Consecutive rows `row0..` of one column (a family run's commit),
605    /// each with its format.
606    Run {
607        seq: u64,
608        sheet_id: SheetId,
609        row0: u32,
610        col0: u32,
611        entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
612    },
613}
614
615impl ComputedWrite {
616    #[inline]
617    pub(crate) fn seq(&self) -> u64 {
618        match self {
619            ComputedWrite::Cell { seq, .. }
620            | ComputedWrite::Rect { seq, .. }
621            | ComputedWrite::Run { seq, .. } => *seq,
622        }
623    }
624}
625
626#[derive(Debug, Default)]
627pub(crate) struct ComputedWriteBuffer {
628    writes: Vec<ComputedWrite>,
629    next_seq: u64,
630    estimated_bytes: usize,
631    formats_present: bool,
632}
633
634impl ComputedWriteBuffer {
635    const ENTRY_BASE_BYTES: usize = 32;
636
637    #[inline]
638    pub(crate) fn is_empty(&self) -> bool {
639        self.writes.is_empty()
640    }
641
642    #[inline]
643    pub(crate) fn len(&self) -> usize {
644        self.writes.len()
645    }
646
647    #[inline]
648    pub(crate) fn writes(&self) -> &[ComputedWrite] {
649        &self.writes
650    }
651
652    #[inline]
653    pub(crate) fn estimated_bytes(&self) -> usize {
654        self.estimated_bytes
655    }
656
657    #[cfg(feature = "tracing")]
658    fn traced_cell_count(&self) -> usize {
659        self.writes
660            .iter()
661            .map(|write| match write {
662                ComputedWrite::Cell { .. } => 1,
663                ComputedWrite::Rect { values, .. } => values.iter().map(Vec::len).sum(),
664                ComputedWrite::Run { entries, .. } => entries.len(),
665            })
666            .sum()
667    }
668
669    pub(crate) fn push_cell(
670        &mut self,
671        sheet_id: SheetId,
672        row0: u32,
673        col0: u32,
674        value: OverlayValue,
675    ) {
676        self.push_cell_with_format(sheet_id, row0, col0, value, None);
677    }
678
679    pub(crate) fn push_cell_with_format(
680        &mut self,
681        sheet_id: SheetId,
682        row0: u32,
683        col0: u32,
684        value: OverlayValue,
685        format_id: Option<crate::format::FormatId>,
686    ) {
687        let format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
688        self.formats_present |= format_id.is_some();
689        let seq = self.next_sequence();
690        self.estimated_bytes = self
691            .estimated_bytes
692            .saturating_add(Self::estimate_value_bytes(&value));
693        self.writes.push(ComputedWrite::Cell {
694            seq,
695            sheet_id,
696            row0,
697            col0,
698            value,
699            format_id,
700        });
701    }
702
703    pub(crate) fn push_column_run(
704        &mut self,
705        sheet_id: SheetId,
706        row0: u32,
707        col0: u32,
708        mut entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
709    ) {
710        let seq = self.next_sequence();
711        let mut added = 0usize;
712        for (value, format_id) in entries.iter_mut() {
713            *format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
714            self.formats_present |= format_id.is_some();
715            added = added.saturating_add(Self::estimate_value_bytes(value));
716        }
717        self.estimated_bytes = self.estimated_bytes.saturating_add(added);
718        self.writes.push(ComputedWrite::Run {
719            seq,
720            sheet_id,
721            row0,
722            col0,
723            entries,
724        });
725    }
726
727    pub(crate) fn push_rect(
728        &mut self,
729        sheet_id: SheetId,
730        sr0: u32,
731        sc0: u32,
732        values: Vec<Vec<OverlayValue>>,
733    ) {
734        let seq = self.next_sequence();
735        let added = values
736            .iter()
737            .flat_map(|row| row.iter())
738            .map(Self::estimate_value_bytes)
739            .fold(0usize, usize::saturating_add);
740        self.estimated_bytes = self.estimated_bytes.saturating_add(added);
741        self.writes.push(ComputedWrite::Rect {
742            seq,
743            sheet_id,
744            sr0,
745            sc0,
746            values,
747        });
748    }
749
750    pub(crate) fn clear(&mut self) {
751        self.writes.clear();
752        self.estimated_bytes = 0;
753        self.formats_present = false;
754    }
755
756    fn take_writes(&mut self) -> (Vec<ComputedWrite>, bool) {
757        self.estimated_bytes = 0;
758        let formats_present = std::mem::take(&mut self.formats_present);
759        (std::mem::take(&mut self.writes), formats_present)
760    }
761
762    fn next_sequence(&mut self) -> u64 {
763        let seq = self.next_seq;
764        self.next_seq = self.next_seq.wrapping_add(1);
765        seq
766    }
767
768    #[inline]
769    fn estimate_value_bytes(value: &OverlayValue) -> usize {
770        Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
771    }
772}
773
774#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
775struct ComputedWriteChunkKey {
776    sheet_id: SheetId,
777    col0: u32,
778    chunk_idx: usize,
779    chunk_start_row0: u32,
780}
781
782#[derive(Debug, Clone, PartialEq)]
783pub(crate) struct ComputedWriteChunkEntryPlan {
784    pub(crate) row_in_chunk: usize,
785    pub(crate) seq: u64,
786    pub(crate) value: OverlayValue,
787    pub(crate) format_id: Option<crate::format::FormatId>,
788}
789
790#[derive(Debug, Clone, PartialEq, Eq)]
791pub(crate) enum ComputedWriteChunkPlanShape {
792    Point,
793    SparseOffsets {
794        entries: usize,
795        span_len: usize,
796    },
797    DenseRange {
798        start: usize,
799        len: usize,
800    },
801    RunRange {
802        start: usize,
803        len: usize,
804        runs: usize,
805    },
806}
807
808#[derive(Debug, Clone, PartialEq, Eq)]
809pub(crate) enum ComputedWriteFormatClear {
810    Range { start: usize, end: usize },
811    Offsets(Vec<usize>),
812}
813
814#[derive(Debug, Clone, PartialEq)]
815pub(crate) enum ComputedWriteChunkFormatEffect {
816    NoFormatWork,
817    ClearStale(ComputedWriteFormatClear),
818    SetExplicit(Vec<(usize, Option<crate::format::FormatId>)>),
819}
820
821#[derive(Debug, Clone, PartialEq)]
822pub(crate) struct ComputedWriteChunkPlan {
823    pub(crate) sheet_id: SheetId,
824    pub(crate) col0: u32,
825    pub(crate) chunk_idx: usize,
826    pub(crate) chunk_start_row0: u32,
827    pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
828    pub(crate) shape: ComputedWriteChunkPlanShape,
829    pub(crate) format_effect: ComputedWriteChunkFormatEffect,
830}
831
832#[derive(Debug, Clone, Default, PartialEq)]
833pub(crate) struct ComputedWriteCoalescingPlan {
834    pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
835    pub(crate) input_cells: usize,
836    pub(crate) coalesced_cells: usize,
837    pub(crate) overwritten_cells: usize,
838}
839
840impl ComputedWriteCoalescingPlan {
841    #[inline]
842    pub(crate) fn is_empty(&self) -> bool {
843        self.chunks.is_empty()
844    }
845}
846
847impl ComputedWriteChunkPlan {
848    fn from_group(
849        key: ComputedWriteChunkKey,
850        mut entries: Vec<ComputedWriteChunkEntryPlan>,
851        formats_present: bool,
852        computed_lane_has_formats: bool,
853    ) -> (Self, usize) {
854        entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
855        let input_len = entries.len();
856        let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
857        for entry in entries {
858            if let Some(prev) = coalesced.last_mut()
859                && prev.row_in_chunk == entry.row_in_chunk
860            {
861                *prev = entry;
862                continue;
863            }
864            coalesced.push(entry);
865        }
866        let overwritten = input_len.saturating_sub(coalesced.len());
867        let shape = Self::classify_shape(&coalesced);
868        let format_effect = Self::classify_format_effect(
869            &coalesced,
870            &shape,
871            formats_present,
872            computed_lane_has_formats,
873        );
874        (
875            Self {
876                sheet_id: key.sheet_id,
877                col0: key.col0,
878                chunk_idx: key.chunk_idx,
879                chunk_start_row0: key.chunk_start_row0,
880                entries: coalesced,
881                shape,
882                format_effect,
883            },
884            overwritten,
885        )
886    }
887
888    fn classify_format_effect(
889        entries: &[ComputedWriteChunkEntryPlan],
890        shape: &ComputedWriteChunkPlanShape,
891        formats_present: bool,
892        computed_lane_has_formats: bool,
893    ) -> ComputedWriteChunkFormatEffect {
894        if !formats_present {
895            return if computed_lane_has_formats {
896                ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
897            } else {
898                ComputedWriteChunkFormatEffect::NoFormatWork
899            };
900        }
901
902        if entries.iter().all(|entry| entry.format_id.is_none()) {
903            return if computed_lane_has_formats {
904                ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
905            } else {
906                ComputedWriteChunkFormatEffect::NoFormatWork
907            };
908        }
909
910        ComputedWriteChunkFormatEffect::SetExplicit(
911            entries
912                .iter()
913                .map(|entry| (entry.row_in_chunk, entry.format_id))
914                .collect(),
915        )
916    }
917
918    fn format_clear_spec(
919        entries: &[ComputedWriteChunkEntryPlan],
920        shape: &ComputedWriteChunkPlanShape,
921    ) -> ComputedWriteFormatClear {
922        match shape {
923            ComputedWriteChunkPlanShape::DenseRange { start, len }
924            | ComputedWriteChunkPlanShape::RunRange { start, len, .. } => {
925                ComputedWriteFormatClear::Range {
926                    start: *start,
927                    end: start.saturating_add(*len),
928                }
929            }
930            ComputedWriteChunkPlanShape::Point
931            | ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
932                ComputedWriteFormatClear::Offsets(
933                    entries.iter().map(|entry| entry.row_in_chunk).collect(),
934                )
935            }
936        }
937    }
938
939    fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
940        debug_assert!(!entries.is_empty());
941        if entries.len() == 1 {
942            return ComputedWriteChunkPlanShape::Point;
943        }
944
945        let start = entries[0].row_in_chunk;
946        let end = entries[entries.len() - 1].row_in_chunk;
947        let span_len = end.saturating_sub(start).saturating_add(1);
948        if span_len != entries.len() {
949            return ComputedWriteChunkPlanShape::SparseOffsets {
950                entries: entries.len(),
951                span_len,
952            };
953        }
954
955        let runs = Self::run_count(entries);
956        if runs < entries.len() {
957            ComputedWriteChunkPlanShape::RunRange {
958                start,
959                len: entries.len(),
960                runs,
961            }
962        } else {
963            ComputedWriteChunkPlanShape::DenseRange {
964                start,
965                len: entries.len(),
966            }
967        }
968    }
969
970    fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
971        let mut runs = 0usize;
972        let mut prev: Option<&OverlayValue> = None;
973        for entry in entries {
974            if prev != Some(&entry.value) {
975                runs = runs.saturating_add(1);
976                prev = Some(&entry.value);
977            }
978        }
979        runs
980    }
981}
982
983#[cfg(feature = "tracing")]
984#[derive(Default)]
985struct TraceEvaluationCounters {
986    computed_vertices: usize,
987    cycles: usize,
988}
989
990pub struct Engine<R> {
991    pub(crate) graph: DependencyGraph,
992    resolver: R,
993    pub config: EvalConfig,
994    workbook_load_limits: crate::engine::WorkbookLoadLimits,
995    /// Clock for volatile date/time builtins, wrapped in a per-recalc
996    /// snapshot: sampled once at the start of every evaluation request
997    /// ([`Self::begin_evaluation_request`]) so all `NOW()`/`TODAY()` reads in
998    /// one recalc — including SCC iteration passes — agree (spec §7.11).
999    clock: crate::timezone::SnapshotClock,
1000    thread_pool: Option<Arc<rayon::ThreadPool>>,
1001    pub recalc_epoch: u64,
1002    snapshot_id: std::sync::atomic::AtomicU64,
1003    topology_epoch: u64,
1004    cached_static_schedule: Option<CachedScheduleEntry>,
1005    /// Program 3 (plan reuse): schedules of recent earlier requests, most
1006    /// recent last (a user alternating between a few inputs recalculates
1007    /// the same few closures). Bounded by `RECENT_SCHEDULES` entries and
1008    /// `RECENT_SCHEDULE_VERTICES` candidate vertices in total.
1009    recent_schedules: Vec<CachedScheduleEntry>,
1010    /// Program 3 (plan reuse): the largest current schedule seen (usually
1011    /// the first evaluation's). A request it covers, and that is not much
1012    /// smaller, takes its schedule restricted to the request instead of
1013    /// planning (`Schedule::restrict`).
1014    base_schedule: Option<CachedScheduleEntry>,
1015    #[cfg(any(test, feature = "benchmark_internal"))]
1016    recalc_reuse_probe: std::sync::Mutex<RecalcReuseProbe>,
1017    spill_mgr: ShimSpillManager,
1018    /// Arrow-backed storage for sheet values (Phase A)
1019    arrow_sheets: SheetStore,
1020    /// Workbook-local number-format registry.
1021    format_registry: crate::format::FormatRegistry,
1022    /// Derived formula formats keyed by grid position, never graph vertex identity.
1023    derived_formats: crate::engine::derived_formats::DerivedFormats,
1024    #[cfg(test)]
1025    derived_format_operations_for_test: std::sync::atomic::AtomicU64,
1026    #[cfg(test)]
1027    family_members_for_test: std::sync::atomic::AtomicU64,
1028    #[cfg(test)]
1029    invariant_bound_members_for_test: std::sync::atomic::AtomicU64,
1030    #[cfg(test)]
1031    lifted_members_for_test: std::sync::atomic::AtomicU64,
1032    #[cfg(test)]
1033    chained_members_for_test: std::sync::atomic::AtomicU64,
1034    #[cfg(test)]
1035    lane_clean_reads_for_test: std::sync::atomic::AtomicU64,
1036    #[cfg(test)]
1037    criteria_kernel_members_for_test: std::sync::atomic::AtomicU64,
1038    #[cfg(test)]
1039    memo_hits_for_test: std::sync::atomic::AtomicU64,
1040    /// Authority build last compressed (`maybe_compress_formulas`).
1041    compressed_at_build: Option<u64>,
1042    #[cfg(test)]
1043    computed_overlay_set_explicit_entry_operations_for_test: u64,
1044    #[cfg(test)]
1045    computed_overlay_stale_clear_range_effects_for_test: u64,
1046    #[cfg(test)]
1047    computed_overlay_stale_clear_offset_attempts_for_test: u64,
1048    #[cfg(test)]
1049    computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64,
1050    /// True if any edit after bulk load; disables Arrow reads for parity
1051    has_edited: bool,
1052    /// Overlay compaction counter (Phase C instrumentation)
1053    overlay_compactions: u64,
1054
1055    // Overlay memory observability / budget (ticket 503)
1056    computed_overlay_bytes_estimate: usize,
1057    computed_overlay_mirroring_disabled: bool,
1058    /// When true, RangeView resolution materializes from graph/Arrow base per-cell.
1059    /// This preserves correctness if we stop mirroring formula/spill outputs into computed overlays.
1060    pub(crate) force_materialize_range_views: bool,
1061    // Pass-scoped cache for Arrow used-row bounds per column
1062    row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
1063    // Snapshot-scoped final used-axis bounds for open-ended references.
1064    used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
1065    lookup_index_cache: LookupIndexCache,
1066    source_cache: Arc<std::sync::RwLock<SourceCache>>,
1067    /// Identity binding for opaque source-family preparations.
1068    source_formula_token: Arc<()>,
1069    /// Dedicated identity binding for reusable recalculation plans.
1070    recalc_plan_token: Arc<()>,
1071    /// Staged formulas by sheet when `defer_graph_building` is enabled.
1072    staged_formulas: StagedFormulaMap,
1073    /// Presence and generation authority for ordinary staged formula discovery.
1074    staged_formula_index: StagedFormulaIndex,
1075    // Occupancy invalidation only: never a formula/read dependency.
1076    blocked_pending_spills: Vec<(VertexId, CellRef, Region)>,
1077    /// Anchors whose spill is blocked by a formula or by another spill's
1078    /// cell, keyed by that blocking cell: a change to the cell (an edit, or
1079    /// the other spill clearing) wakes them. Never a read dependency.
1080    spill_blocker_waiters: SpillBlockerWaiters,
1081    /// Per-sheet row visibility sidecar state.
1082    row_visibility: FxHashMap<SheetId, RowVisibilityState>,
1083    /// Cached row visibility masks keyed by sheet/span/mode/version.
1084    row_visibility_mask_cache: std::sync::RwLock<
1085        FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
1086    >,
1087    /// Non-fatal malformed formula diagnostics captured during ingest/graph-build.
1088    formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
1089    /// Last centralized formula ingest report.
1090    last_formula_ingest_report: Option<FormulaIngestReport>,
1091    /// Aggregate centralized formula ingest report for this engine.
1092    formula_ingest_report_total: FormulaIngestReport,
1093    /// Transient cancellation flag used during evaluation
1094    active_cancel_flag: Option<crate::engine::CancelToken>,
1095    /// Transient absolute deadline used by composed target and plan requests.
1096    active_evaluation_deadline: Option<Instant>,
1097
1098    /// Engine-level action depth.
1099    ///
1100    /// Ticket 614 introduces `Engine::action` as a stable, commit-only transaction surface.
1101    /// Nested actions are currently disallowed (deterministic rule) and will return an error.
1102    action_depth: u32,
1103
1104    // Phase 3b virtual-dependency convergence telemetry
1105    last_virtual_dep_telemetry: VirtualDepTelemetry,
1106    virtual_dep_fallback_activations: u64,
1107
1108    // Runtime-cycle SCC evaluation telemetry (RFC #112, Stage 2)
1109    last_cycle_telemetry: CycleTelemetry,
1110
1111    // C0 evaluation-resource observability. IDs are never reset or reused.
1112    next_evaluation_resource_request_id: u64,
1113    evaluation_resource_request_depth: usize,
1114    active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1115    last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1116    evaluation_resource_baseline: EvaluationResourceBaselineStats,
1117    evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1118    evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1119    evaluation_resource_config_diagnostic:
1120        Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1121    active_resource_ledger: Option<ResourceLedger>,
1122    source_cache_footprints: Vec<std::sync::Weak<std::sync::atomic::AtomicU64>>,
1123    source_cache_accounted: u64,
1124
1125    /// SCC members that entered iterative calculation (`CyclePolicy::Iterate`
1126    /// with a witnessed live cycle) during the current evaluation request
1127    /// and must re-run on the next one.
1128    ///
1129    /// Excel re-evaluates circular cells on EVERY recalc (the accumulator
1130    /// contract, spec §4/§7.6), but this engine's dirty model marks SCC
1131    /// members clean after a recalc and would otherwise skip them forever.
1132    /// Resolution: members of iterating SCCs are redirtied volatile-like at
1133    /// the end of the same recalc that iterated them
1134    /// ([`Self::redirty_for_next_recalc`], called wherever
1135    /// `redirty_volatiles` runs). The set is per-recalc, never persisted:
1136    /// if an edit breaks the cycle, the next recalc's SCC task either does
1137    /// not exist or settles as phantom, nothing re-registers, and the
1138    /// redirty chain stops by itself.
1139    ///
1140    /// SCCs that landed on an exact fixed point are exempt: they go to
1141    /// [`Self::retained_scc_members`] instead and stay clean until the dirty
1142    /// graph (or a config change) reaches them (#368).
1143    pending_iterative_redirty: Vec<VertexId>,
1144    /// Opt-in snapshot request: defer only next-cycle volatile redirty.
1145    snapshot_evaluation_active: bool,
1146    snapshot_volatile_redirty_pending: bool,
1147    /// Members of iterating SCCs retained across recalcs (#368), keyed to the
1148    /// id of the retained SCC they belong to (ids come from
1149    /// `next_retained_scc_id`; grouping is only used for telemetry).
1150    ///
1151    /// An SCC is retained when the recalc that iterated it stopped because
1152    /// every member reproduced its previous value exactly (|Δ| = 0, or
1153    /// identity for non-numeric members; never NaN-converged), before the
1154    /// `max_iterations` cap, with no volatile or dynamic-reference member.
1155    /// Such an SCC is a fixed point of its own inputs: running it again with
1156    /// the same inputs cannot change any value, so it is not redirtied. The
1157    /// dirty graph remains the validity authority — any edit that reaches a
1158    /// member dirties it like any other formula and the SCC task re-runs.
1159    /// Membership is dropped when a member runs in an SCC task again (it is
1160    /// then re-retained or re-registered for per-recalc redirty), when the
1161    /// vertex is deleted, or when
1162    /// [`Self::reconcile_retained_sccs_at_request_begin`] invalidates the
1163    /// whole set because a config knob outside the graph changed.
1164    retained_scc_members: FxHashMap<VertexId, u64>,
1165    next_retained_scc_id: u64,
1166    /// [`Self::retained_scc_config_fingerprint`] as of the last retention.
1167    /// `Engine::config` is a public field, so knobs that change a retained
1168    /// SCC's result (cycle policy/tolerance, date system, determinism,
1169    /// volatile seeding) can change between recalcs without touching the
1170    /// graph; a mismatch at request begin dirties every retained member.
1171    /// Only meaningful while `retained_scc_members` is non-empty.
1172    retained_scc_config_fingerprint: u64,
1173    /// Function-registry semantic epoch and runtime-provider revision as of
1174    /// the last time retained SCCs were reconciled against them. A newer
1175    /// epoch dirties only the retained members whose formula calls a changed
1176    /// function (or every member when the change log is incomplete).
1177    retained_scc_function_epoch_seen: u64,
1178    retained_scc_provider_revision_seen: Option<u64>,
1179    /// Retained members that were already dirty when the current request
1180    /// began, with the SCC id they carried. A member still holding that id
1181    /// at the end of the request was not touched by any SCC task — its
1182    /// cycle dissolved (it evaluated as an ordinary formula) or the request
1183    /// never reached it — so [`Self::redirty_for_next_recalc`] drops it from
1184    /// the retained set instead of letting it linger.
1185    retained_scc_dirty_at_begin: Vec<(VertexId, u64)>,
1186
1187    /// Final committed values of iterating-SCC members (spec §4 persistence).
1188    /// In canonical (value-cache disabled) mode the computed overlay is the
1189    /// ONLY home of a formula's value, and structural edits clear computed
1190    /// overlays wholesale (`clear_computed_overlay_after_row/_col`) —
1191    /// destroying iteration state (accumulators reset to 0; found by the
1192    /// iterate edge corpus). This snapshot lets the next SCC task re-seed
1193    /// members whose overlay entry vanished. Members registered for
1194    /// per-recalc redirty are refreshed by
1195    /// [`Self::redirty_for_next_recalc`]; retained members are written once
1196    /// when retained. Entries are dropped when the member's SCC task ends
1197    /// without iterating or when the vertex is deleted. Empty unless
1198    /// something iterated — zero cost otherwise.
1199    iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1200
1201    /// Global function-registry semantic epoch last observed.
1202    function_semantic_epoch_seen: u64,
1203    /// Runtime-provider semantic revision last observed.
1204    function_provider_revision_seen: Option<u64>,
1205
1206    #[cfg(feature = "tracing")]
1207    trace_evaluation_counters: TraceEvaluationCounters,
1208    #[cfg(test)]
1209    evaluation_request_begin_count_for_test: u64,
1210    #[cfg(any(test, feature = "test-support"))]
1211    before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1212    #[cfg(test)]
1213    before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1214    #[cfg(test)]
1215    before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1216    #[cfg(test)]
1217    inject_target_semantic_stale_once_for_test: bool,
1218    #[cfg(test)]
1219    force_virtual_dep_changes_remaining_for_test: usize,
1220    /// Dynamic-read freshness state (design §8.2).
1221    freshness: freshness::Freshness,
1222    #[cfg(test)]
1223    fail_evaluation_commit_preflight_once_for_test: bool,
1224    #[cfg(test)]
1225    target_preparation_fault_for_test:
1226        Option<crate::engine::target_preparation::TargetPreparationFault>,
1227    #[cfg(test)]
1228    force_non_cycle_schedule_fallback_for_test: bool,
1229    #[cfg(test)]
1230    before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1231    #[cfg(test)]
1232    after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1233}
1234
1235/// This wrapper is intentionally thin for ticket 614 (commit-only): it delegates to existing
1236/// `Engine` edit methods and does not create changelog boundaries or implement rollback.
1237impl<R: EvaluationContext> Engine<R> {
1238    pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1239        self.graph.ingest_pipeline(&self.resolver)
1240    }
1241}
1242
1243pub struct EngineAction<'a, R>
1244where
1245    R: EvaluationContext,
1246{
1247    engine: &'a mut Engine<R>,
1248    name: String,
1249    // Complete private mutation capture used by atomic actions.
1250    // Stored as a raw pointer to avoid creating aliasing `&mut` borrows alongside `&mut Engine`.
1251    capture: Option<*mut MutationCapture>,
1252    // Optional Arrow undo journal used by `Engine::action_atomic`.
1253    // Stored as a raw pointer to avoid aliasing issues with `&mut Engine`.
1254    arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1255    // True when this EngineAction must enforce conservative atomic transaction policy.
1256    atomic_policy: bool,
1257}
1258
1259impl<'a, R> EngineAction<'a, R>
1260where
1261    R: EvaluationContext,
1262{
1263    #[inline]
1264    fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1265        let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1266        let coord = crate::reference::Coord::from_excel(row, col, true, true);
1267        crate::reference::CellRef::new(sheet_id, coord)
1268    }
1269
1270    #[inline]
1271    pub fn name(&self) -> &str {
1272        &self.name
1273    }
1274
1275    #[inline]
1276    pub fn set_cell_value(
1277        &mut self,
1278        sheet: &str,
1279        row: u32,
1280        col: u32,
1281        value: LiteralValue,
1282    ) -> Result<(), crate::engine::EditorError> {
1283        if self.capture.is_some() {
1284            let old_value = self.engine.read_cell_value(sheet, row, col);
1285            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1286            let addr = self.addr_for(sheet, row, col);
1287            let Some(capture_ptr) = self.capture else {
1288                return Err(crate::engine::EditorError::TransactionFailed {
1289                    reason: "action_with_logger: missing mutation capture".to_string(),
1290                });
1291            };
1292
1293            // For atomic journal mode, record computed overlay effects for this cell.
1294            // Delta-overlay undo is recorded semantically based on old_value/old_formula.
1295            let old_comp = if self.arrow_undo.is_some() {
1296                self.engine.read_computed_overlay_cell(sheet, row, col)
1297            } else {
1298                None
1299            };
1300
1301            if self.engine.graph_admission_enabled() {
1302                let admission =
1303                    self.engine
1304                        .graph
1305                        .preview_value_mutation(addr.sheet_id, row, col)?;
1306                self.engine.preflight_graph_admission(admission)?;
1307            }
1308            if old_formula.is_none() {
1309                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1310            }
1311
1312            let delta_old_sem = if old_formula.is_some() {
1313                None
1314            } else {
1315                Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1316            };
1317
1318            let start_len = unsafe { (&*capture_ptr).len() };
1319
1320            // Safety: `capture_ptr` comes from a unique operation-local `&mut MutationCapture`.
1321            let capture = unsafe { &mut *capture_ptr };
1322            self.engine.edit_with_capture(capture, |editor| {
1323                editor.set_cell_value_with_old_state(
1324                    addr,
1325                    value.clone(),
1326                    old_value.clone(),
1327                    old_formula.clone(),
1328                );
1329            })?;
1330            self.engine.record_structural_change(StructuralScope::Cell {
1331                sheet: addr.sheet_id,
1332                row: addr.coord.row(),
1333                col: addr.coord.col(),
1334            });
1335
1336            if let Some(undo_ptr) = self.arrow_undo {
1337                // 1) Spill snapshot operations (computed overlay rect restore).
1338                let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1339                let undo = unsafe { &mut *undo_ptr };
1340                self.engine
1341                    .record_spill_ops_into_arrow_undo(undo, new_events);
1342
1343                // 2) Delta/computed overlay single-cell deltas.
1344                let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1345                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1346                let row0 = row.saturating_sub(1);
1347                let col0 = col.saturating_sub(1);
1348                let delta_new_sem = Some(value.clone());
1349                undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1350                undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1351            }
1352            Ok(())
1353        } else {
1354            self.engine
1355                .set_cell_value(sheet, row, col, value)
1356                .map_err(crate::engine::EditorError::from)
1357        }
1358    }
1359
1360    #[inline]
1361    pub fn set_cell_formula(
1362        &mut self,
1363        sheet: &str,
1364        row: u32,
1365        col: u32,
1366        ast: ASTNode,
1367    ) -> Result<(), crate::engine::EditorError> {
1368        if self.capture.is_some() {
1369            let old_value = self.engine.read_cell_value(sheet, row, col);
1370            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1371            let addr = self.addr_for(sheet, row, col);
1372            let Some(capture_ptr) = self.capture else {
1373                return Err(crate::engine::EditorError::TransactionFailed {
1374                    reason: "action_with_logger: missing mutation capture".to_string(),
1375                });
1376            };
1377
1378            let admitted_formula = if self.engine.graph_admission_enabled() {
1379                let placement =
1380                    CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1381                let ingested = self.engine.ingest_pipeline().ingest_formula(
1382                    FormulaAstInput::Tree(ast.clone()),
1383                    placement,
1384                    None,
1385                )?;
1386                let admission = self.engine.graph.preview_formula_mutations(&[(
1387                    addr.sheet_id,
1388                    row,
1389                    col,
1390                    ingested.dep_plan.clone(),
1391                )])?;
1392                self.engine.preflight_graph_admission(admission)?;
1393                Some((ingested.ast_id, ingested.dep_plan))
1394            } else {
1395                None
1396            };
1397            if old_formula.is_none() {
1398                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1399            }
1400            let delta_old = if self.arrow_undo.is_some() {
1401                if old_formula.is_some() {
1402                    None
1403                } else {
1404                    Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1405                }
1406            } else {
1407                None
1408            };
1409            let start_len = unsafe { (&*capture_ptr).len() };
1410
1411            // Safety: `capture_ptr` comes from a unique operation-local `&mut MutationCapture`.
1412            let capture = unsafe { &mut *capture_ptr };
1413            self.engine.edit_with_capture(capture, |editor| {
1414                if let Some((ast_id, plan)) = admitted_formula {
1415                    editor.set_cell_formula_with_prepared_plan(
1416                        addr,
1417                        ast.clone(),
1418                        old_value,
1419                        old_formula,
1420                        ast_id,
1421                        plan,
1422                    );
1423                } else {
1424                    editor.set_cell_formula_with_old_state(
1425                        addr,
1426                        ast.clone(),
1427                        old_value,
1428                        old_formula,
1429                    );
1430                }
1431            })?;
1432            self.engine.record_structural_change(StructuralScope::Cell {
1433                sheet: addr.sheet_id,
1434                row: addr.coord.row(),
1435                col: addr.coord.col(),
1436            });
1437
1438            if let Some(undo_ptr) = self.arrow_undo {
1439                let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1440                let undo = unsafe { &mut *undo_ptr };
1441                self.engine
1442                    .record_spill_ops_into_arrow_undo(undo, new_events);
1443                let delta_new: Option<LiteralValue> = None;
1444                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1445                let row0 = row.saturating_sub(1);
1446                let col0 = col.saturating_sub(1);
1447                undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1448            }
1449            Ok(())
1450        } else {
1451            self.engine
1452                .set_cell_formula(sheet, row, col, ast)
1453                .map_err(crate::engine::EditorError::from)
1454        }
1455    }
1456
1457    #[inline]
1458    pub fn set_row_hidden(
1459        &mut self,
1460        sheet: &str,
1461        row_1based: u32,
1462        hidden: bool,
1463        source: RowVisibilitySource,
1464    ) -> Result<(), crate::engine::EditorError> {
1465        if self.capture.is_some() {
1466            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1467            let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1468            let old_hidden = self
1469                .engine
1470                .row_visibility
1471                .get(&sheet_id)
1472                .map(|state| state.is_row_hidden(row0, Some(source)))
1473                .unwrap_or(false);
1474            if old_hidden == hidden {
1475                return Ok(());
1476            }
1477
1478            let _ = self
1479                .engine
1480                .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1481
1482            let Some(capture_ptr) = self.capture else {
1483                return Err(crate::engine::EditorError::TransactionFailed {
1484                    reason: "action_with_logger: missing mutation capture".to_string(),
1485                });
1486            };
1487            unsafe { &mut *capture_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1488                sheet_id,
1489                row0,
1490                source,
1491                old_hidden,
1492                new_hidden: hidden,
1493            });
1494
1495            Ok(())
1496        } else {
1497            self.engine
1498                .set_row_hidden(sheet, row_1based, hidden, source)
1499        }
1500    }
1501
1502    #[inline]
1503    pub fn set_rows_hidden(
1504        &mut self,
1505        sheet: &str,
1506        start_row_1based: u32,
1507        end_row_1based: u32,
1508        hidden: bool,
1509        source: RowVisibilitySource,
1510    ) -> Result<(), crate::engine::EditorError> {
1511        if self.capture.is_some() {
1512            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1513            let (start_row0, end_row0) =
1514                Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1515
1516            let Some(capture_ptr) = self.capture else {
1517                return Err(crate::engine::EditorError::TransactionFailed {
1518                    reason: "action_with_logger: missing mutation capture".to_string(),
1519                });
1520            };
1521            let capture = unsafe { &mut *capture_ptr };
1522
1523            for row0 in start_row0..=end_row0 {
1524                let old_hidden = self
1525                    .engine
1526                    .row_visibility
1527                    .get(&sheet_id)
1528                    .map(|state| state.is_row_hidden(row0, Some(source)))
1529                    .unwrap_or(false);
1530                if old_hidden == hidden {
1531                    continue;
1532                }
1533
1534                let _ = self
1535                    .engine
1536                    .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1537
1538                capture.record(crate::engine::ChangeEvent::SetRowVisibility {
1539                    sheet_id,
1540                    row0,
1541                    source,
1542                    old_hidden,
1543                    new_hidden: hidden,
1544                });
1545            }
1546
1547            Ok(())
1548        } else {
1549            self.engine
1550                .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1551        }
1552    }
1553
1554    #[inline]
1555    pub fn insert_rows(
1556        &mut self,
1557        sheet: &str,
1558        before: u32,
1559        count: u32,
1560    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1561        if count == 0 {
1562            return Ok(crate::engine::ShiftSummary::default());
1563        }
1564        if self.capture.is_some() {
1565            let Some(capture_ptr) = self.capture else {
1566                return Err(crate::engine::EditorError::TransactionFailed {
1567                    reason: "action_atomic: missing mutation capture".to_string(),
1568                });
1569            };
1570
1571            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1572            let before0 = before.saturating_sub(1);
1573            let occupancy = self.engine.structural_row_occupancy(sheet, sheet_id);
1574            let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1575
1576            // Graph structural insert (logged) - no snapshot bump.
1577            let summary = {
1578                let capture = unsafe { &mut *capture_ptr };
1579                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1580                    Ok(crate::engine::ShiftSummary::default());
1581                self.engine.edit_with_capture(capture, |editor| {
1582                    editor.set_structural_occupancy(occupancy);
1583                    out = editor.insert_rows(sheet_id, before0, count);
1584                })?;
1585                out?
1586            };
1587
1588            // Arrow insert (truth) + undo op.
1589            self.engine.ensure_arrow_sheet(sheet);
1590            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1591                asheet.insert_rows(before0 as usize, count as usize);
1592            }
1593            self.engine
1594                .purge_derived_formats_after_row(sheet_id, before0);
1595            self.engine
1596                .shift_row_visibility_insert(sheet_id, before0, count);
1597            self.engine.mark_moved_formula_vertices_dirty(&summary);
1598            self.engine
1599                .clear_computed_overlay_after_row(sheet, before0 as usize);
1600            self.engine
1601                .record_structural_change(StructuralScope::Region(affected_region));
1602            if let Some(undo_ptr) = self.arrow_undo {
1603                unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1604            }
1605            Ok(summary)
1606        } else {
1607            self.engine.insert_rows(sheet, before, count)
1608        }
1609    }
1610
1611    #[inline]
1612    pub fn delete_rows(
1613        &mut self,
1614        sheet: &str,
1615        start: u32,
1616        count: u32,
1617    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1618        if count == 0 {
1619            return Ok(crate::engine::ShiftSummary::default());
1620        }
1621        if self.atomic_policy {
1622            return Err(crate::engine::EditorError::TransactionUnsupported {
1623                reason:
1624                    "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1625                        .to_string(),
1626            });
1627        }
1628        self.engine.delete_rows(sheet, start, count)
1629    }
1630
1631    #[inline]
1632    pub fn insert_columns(
1633        &mut self,
1634        sheet: &str,
1635        before: u32,
1636        count: u32,
1637    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1638        if count == 0 {
1639            return Ok(crate::engine::ShiftSummary::default());
1640        }
1641        if self.capture.is_some() {
1642            let Some(capture_ptr) = self.capture else {
1643                return Err(crate::engine::EditorError::TransactionFailed {
1644                    reason: "action_atomic: missing mutation capture".to_string(),
1645                });
1646            };
1647
1648            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1649            let before0 = before.saturating_sub(1);
1650            let occupancy = self.engine.structural_column_occupancy();
1651            let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1652
1653            let summary = {
1654                let capture = unsafe { &mut *capture_ptr };
1655                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1656                    Ok(crate::engine::ShiftSummary::default());
1657                self.engine.edit_with_capture(capture, |editor| {
1658                    editor.set_structural_occupancy(occupancy);
1659                    out = editor.insert_columns(sheet_id, before0, count);
1660                })?;
1661                out?
1662            };
1663
1664            self.engine.ensure_arrow_sheet(sheet);
1665            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1666                asheet.insert_columns(before0 as usize, count as usize);
1667            }
1668            self.engine
1669                .purge_derived_formats_after_col(sheet_id, before0);
1670            self.engine.mark_moved_formula_vertices_dirty(&summary);
1671            self.engine
1672                .clear_computed_overlay_after_col(sheet, before0 as usize);
1673            self.engine
1674                .record_structural_change(StructuralScope::Region(affected_region));
1675            if let Some(undo_ptr) = self.arrow_undo {
1676                unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1677            }
1678            Ok(summary)
1679        } else {
1680            self.engine.insert_columns(sheet, before, count)
1681        }
1682    }
1683
1684    #[inline]
1685    pub fn delete_columns(
1686        &mut self,
1687        sheet: &str,
1688        start: u32,
1689        count: u32,
1690    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1691        if count == 0 {
1692            return Ok(crate::engine::ShiftSummary::default());
1693        }
1694        if self.atomic_policy {
1695            return Err(crate::engine::EditorError::TransactionUnsupported {
1696                reason:
1697                    "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1698                        .to_string(),
1699            });
1700        }
1701        self.engine.delete_columns(sheet, start, count)
1702    }
1703
1704    /// Start an action from within an action.
1705    ///
1706    /// Nested actions are currently disallowed (ticket 614), so this will return a
1707    /// `EditorError::TransactionFailed` while an outer action is active.
1708    #[inline]
1709    pub fn action<T>(
1710        &mut self,
1711        name: impl AsRef<str>,
1712        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1713    ) -> Result<T, crate::engine::EditorError> {
1714        self.engine.action(name, f)
1715    }
1716}
1717
1718struct ActionDepthGuard<'a, R> {
1719    engine: *mut Engine<R>,
1720    _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1721}
1722
1723impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1724    fn drop(&mut self) {
1725        // Safety: the guard is created from a unique `&mut Engine` borrow and lives no longer
1726        // than the surrounding `Engine::action` call.
1727        unsafe {
1728            let e = &mut *self.engine;
1729            e.action_depth = e.action_depth.saturating_sub(1);
1730        }
1731    }
1732}
1733
1734#[derive(Default)]
1735struct SourceCache {
1736    scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1737    tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1738}
1739
1740#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1741struct VisibilityMaskCacheKey {
1742    sheet_id: SheetId,
1743    start_row0: u32,
1744    end_row0: u32,
1745    mode: VisibilityMaskMode,
1746    version: u64,
1747}
1748
1749#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1750enum StructuralScope {
1751    Cell { sheet: SheetId, row: u32, col: u32 },
1752    Region(Region),
1753    Sheet(SheetId),
1754    RemovedSheet(SheetId),
1755    OpaqueGlobal,
1756    AllSheets,
1757}
1758
1759/// Anchors blocked by a formula or another spill, indexed by the blocking
1760/// cell so that a single-cell change finds its waiters in O(1).
1761#[derive(Default)]
1762struct SpillBlockerWaiters {
1763    by_cell: FxHashMap<CellRef, Vec<VertexId>>,
1764    by_anchor: FxHashMap<VertexId, CellRef>,
1765}
1766
1767impl SpillBlockerWaiters {
1768    /// Largest region scanned cell by cell; a larger one scans the waiters.
1769    const REGION_PROBE_CELLS: u64 = 4096;
1770
1771    fn is_empty(&self) -> bool {
1772        self.by_anchor.is_empty()
1773    }
1774
1775    fn wait(&mut self, anchor: VertexId, blocker: CellRef) {
1776        match self.by_anchor.insert(anchor, blocker) {
1777            Some(old) if old == blocker => return,
1778            Some(old) => self.unlink(anchor, old),
1779            None => {}
1780        }
1781        self.by_cell.entry(blocker).or_default().push(anchor);
1782    }
1783
1784    fn forget(&mut self, anchor: VertexId) {
1785        if self.by_anchor.is_empty() {
1786            return;
1787        }
1788        if let Some(old) = self.by_anchor.remove(&anchor) {
1789            self.unlink(anchor, old);
1790        }
1791    }
1792
1793    fn unlink(&mut self, anchor: VertexId, cell: CellRef) {
1794        if let Some(waiters) = self.by_cell.get_mut(&cell) {
1795            waiters.retain(|&v| v != anchor);
1796            if waiters.is_empty() {
1797                self.by_cell.remove(&cell);
1798            }
1799        }
1800    }
1801
1802    /// Remove and return the anchors waiting on a cell inside `scope`.
1803    fn take_affected(&mut self, scope: StructuralScope) -> Vec<VertexId> {
1804        let cells: Vec<CellRef> = match scope {
1805            StructuralScope::Cell { sheet, row, col } => {
1806                let cell = CellRef::new(sheet, Coord::new(row, col, true, true));
1807                if self.by_cell.contains_key(&cell) {
1808                    vec![cell]
1809                } else {
1810                    Vec::new()
1811                }
1812            }
1813            StructuralScope::Region(region) => {
1814                let (rows, cols) = region.axis_ranges();
1815                let (r0, r1) = rows.query_bounds();
1816                let (c0, c1) = cols.query_bounds();
1817                let area = u64::from(r1.saturating_sub(r0) + 1)
1818                    .saturating_mul(u64::from(c1.saturating_sub(c0) + 1));
1819                if area <= Self::REGION_PROBE_CELLS {
1820                    let sheet = region.sheet_id();
1821                    (r0..=r1)
1822                        .flat_map(|r| (c0..=c1).map(move |c| (r, c)))
1823                        .map(|(r, c)| CellRef::new(sheet, Coord::new(r, c, true, true)))
1824                        .filter(|cell| self.by_cell.contains_key(cell))
1825                        .collect()
1826                } else {
1827                    self.by_cell
1828                        .keys()
1829                        .filter(|cell| {
1830                            region.intersects(&Region::point(
1831                                cell.sheet_id,
1832                                cell.coord.row(),
1833                                cell.coord.col(),
1834                            ))
1835                        })
1836                        .copied()
1837                        .collect()
1838                }
1839            }
1840            StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => self
1841                .by_cell
1842                .keys()
1843                .filter(|cell| cell.sheet_id == sheet)
1844                .copied()
1845                .collect(),
1846            StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => {
1847                self.by_cell.keys().copied().collect()
1848            }
1849        };
1850        let mut woken = Vec::new();
1851        for cell in cells {
1852            if let Some(waiters) = self.by_cell.remove(&cell) {
1853                for anchor in waiters {
1854                    self.by_anchor.remove(&anchor);
1855                    woken.push(anchor);
1856                }
1857            }
1858        }
1859        woken
1860    }
1861}
1862
1863#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
1864enum LoggedEditImpact {
1865    NoOp,
1866    DataOnly,
1867    Topology,
1868}
1869
1870#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1871enum LoggedEditDirection {
1872    Original,
1873    InverseReplay,
1874    ForwardReplay,
1875}
1876
1877#[derive(Clone, Copy)]
1878struct InvalidationBaseline {
1879    snapshot_id: u64,
1880    topology_epoch: u64,
1881}
1882
1883struct SourceCacheSession {
1884    cache: Arc<std::sync::RwLock<SourceCache>>,
1885}
1886
1887impl Drop for SourceCacheSession {
1888    fn drop(&mut self) {
1889        if let Ok(mut g) = self.cache.write() {
1890            *g = SourceCache::default();
1891        }
1892    }
1893}
1894
1895#[derive(Debug)]
1896#[non_exhaustive]
1897pub struct EvalResult {
1898    pub computed_vertices: usize,
1899    pub cycle_errors: usize,
1900    pub elapsed: std::time::Duration,
1901}
1902
1903#[derive(Clone, Debug, PartialEq, Eq)]
1904#[non_exhaustive]
1905pub struct TableMetadata {
1906    pub name: String,
1907    pub sheet: String,
1908    pub start_row: u32,
1909    pub start_col: u32,
1910    pub end_row: u32,
1911    pub end_col: u32,
1912    pub header_row: bool,
1913    pub headers: Vec<String>,
1914    pub totals_row: bool,
1915}
1916
1917/// Read-only engine counters used by benchmark/instrumentation tooling.
1918///
1919/// These counters are deliberately observational: collecting them must not mutate engine state or
1920/// alter formula evaluation semantics.
1921///
1922/// The `formula_plane_*` counters are always `0`: FormulaPlane spans were removed and the
1923/// dependency authority is the only runtime path. They are kept for source compatibility.
1924#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1925#[non_exhaustive]
1926pub struct EngineBaselineStats {
1927    pub graph_vertex_count: usize,
1928    pub graph_formula_vertex_count: usize,
1929    pub graph_edge_count: usize,
1930    pub dirty_vertex_count: usize,
1931    pub evaluation_vertex_count: usize,
1932    pub formula_ast_root_count: usize,
1933    pub formula_ast_node_count: usize,
1934    pub staged_formula_count: usize,
1935    pub formula_plane_active_span_count: usize,
1936    pub formula_plane_producer_result_entries: usize,
1937    pub formula_plane_consumer_read_entries: usize,
1938    pub formula_plane_mixed_topology_cache_builds: u64,
1939    pub formula_plane_mixed_topology_cache_hits: u64,
1940    pub formula_plane_mixed_topology_cache_overflows: u64,
1941    pub formula_plane_dirty_pending_events: usize,
1942    pub formula_plane_dirty_region_events_recorded: u64,
1943    pub formula_plane_dirty_span_region_events_recorded: u64,
1944    pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1945    pub formula_plane_dirty_global_invalidations: u64,
1946    pub formula_plane_structural_span_candidates: u64,
1947    pub formula_plane_cycle_member_span_demotions: u64,
1948    pub formula_plane_array_result_span_demotions: u64,
1949    /// Members of exactly converged iterative SCCs currently retained across
1950    /// recalcs (#368).
1951    pub retained_scc_members: usize,
1952}
1953
1954#[derive(Debug, Clone, Default)]
1955#[non_exhaustive]
1956pub struct VirtualDepTelemetry {
1957    pub candidate_vertices_total: usize,
1958    pub vdeps_vertices_total: usize,
1959    pub vdeps_edges_total: usize,
1960    pub builder_elapsed_ms_total: u128,
1961    pub schedule_virtual_passes: usize,
1962    pub schedule_static_passes: usize,
1963    pub schedule_cache_hits: usize,
1964    pub schedule_cache_misses: usize,
1965    pub reused_schedule_vertices_total: usize,
1966    pub replan_iterations: usize,
1967    pub changed_vdeps_total: usize,
1968    pub bailout_reason: Option<&'static str>,
1969    pub fallback_mode_activations: u64,
1970}
1971
1972/// Per-recalc telemetry for SCC evaluation under `CycleDetection::Runtime`
1973/// (spec `formualizer-cycle-semantics-spec.md` §10).
1974///
1975/// Collection is unconditional: SCC tasks are rare relative to ordinary
1976/// vertex evaluation and the counters are a handful of integer adds per
1977/// task, so no config flag gates them (unlike [`VirtualDepTelemetry`],
1978/// which pays per-schedule costs). Counters reset at the start of every
1979/// evaluation request.
1980#[derive(Debug, Clone, Default, PartialEq)]
1981#[non_exhaustive]
1982pub struct CycleTelemetry {
1983    /// SCC tasks executed (static SCCs that reached Runtime evaluation).
1984    pub static_sccs: usize,
1985    /// SCC tasks whose live subgraph was acyclic — values produced.
1986    pub phantom_sccs: usize,
1987    /// Distinct live cycles witnessed across all SCC tasks.
1988    pub live_cycles_witnessed: usize,
1989    /// Cells stamped `#CIRC!` by Runtime SCC tasks.
1990    pub circ_cells_stamped: usize,
1991    /// Evaluation sweeps over (subsets of) SCC members, totalled across tasks
1992    /// (pass 1 included).
1993    pub settle_passes_total: usize,
1994    /// Largest pass count any single SCC task needed.
1995    pub max_passes_single_scc: usize,
1996    /// SCC tasks that entered iterative calculation (`CyclePolicy::Iterate`
1997    /// with a witnessed live cycle). RFC #113, Stage 3.
1998    pub iterated_sccs: usize,
1999    /// Iterating SCC tasks that stopped because every member passed the
2000    /// spec-§6 convergence test.
2001    pub converged_sccs: usize,
2002    /// SCC tasks that stopped at a pass cap. Under `CyclePolicy::Iterate`
2003    /// this is the Excel `max_iterations` cap (NOT an error — last values
2004    /// are kept; includes the no-convergence-test `max_iterations: 1`
2005    /// contract). Under `CyclePolicy::Error` it is the defensive acyclic
2006    /// settle cap (|SCC| + 2), which only a bug can hit.
2007    pub capped_sccs: usize,
2008    /// Largest `|Δ|` observed in any member's final-pass convergence
2009    /// comparison across iterating SCC tasks (numeric-class members only).
2010    /// `0.0` when no comparison ran (e.g. `max_iterations: 1`).
2011    pub max_abs_delta_at_stop: f64,
2012    /// Identical-bit NaN vs NaN member comparisons that were treated as
2013    /// converged (spec §6 NaN rule).
2014    pub nan_converged: usize,
2015    /// Retained iterative SCCs (#368) that had no dirty member at request begin and were therefore not
2016    /// re-run: their last exact fixed point is served as-is. Counted at
2017    /// request begin, so a demand-driven request that never reaches a
2018    /// retained SCC still reports it as reused.
2019    pub reused_sccs: usize,
2020    /// Members of the SCCs counted in `reused_sccs`.
2021    pub reused_scc_members: usize,
2022    /// Total wall-clock time spent inside Runtime SCC tasks.
2023    pub elapsed_ms: u128,
2024}
2025
2026#[derive(Debug, Clone, Copy)]
2027struct ScheduleBuildMeta {
2028    candidate_vertices: usize,
2029    vdeps_vertices: usize,
2030    vdeps_edges: usize,
2031    builder_elapsed_ms: u128,
2032    used_virtual_schedule: bool,
2033    schedule_cache_hit: bool,
2034    schedule_cache_eligible: bool,
2035}
2036
2037#[cfg(any(test, feature = "benchmark_internal"))]
2038#[doc(hidden)]
2039#[derive(Debug, Clone, Default)]
2040pub struct RecalcReuseProbe {
2041    pub schedule_requests: usize,
2042    pub schedule_cache_hits: usize,
2043    pub schedule_cache_misses: usize,
2044    pub schedule_cache_ineligible: usize,
2045    pub schedule_builds: usize,
2046    /// Program 3 plan reuse: misses served by restricting the base schedule.
2047    pub schedule_base_restrictions: usize,
2048    pub schedule_shared_handles: usize,
2049    pub schedule_retained_bytes: usize,
2050    pub legacy_target_requests: usize,
2051    pub target_schedule_builds: usize,
2052    pub demand_builds: usize,
2053    pub demand_vertices: usize,
2054    pub demand_clean_formulas: usize,
2055    pub demand_explicit_edges: usize,
2056    pub demand_virtual_builder_calls: usize,
2057}
2058
2059#[cfg(any(test, feature = "benchmark_internal"))]
2060fn schedule_probe_retained_bytes(schedule: &crate::engine::Schedule) -> usize {
2061    fn vector_bytes<T>(values: &Vec<T>) -> usize {
2062        values.capacity() * std::mem::size_of::<T>()
2063    }
2064
2065    [
2066        vector_bytes(&schedule.units),
2067        vector_bytes(&schedule.layers),
2068        vector_bytes(&schedule.cycles),
2069    ]
2070    .into_iter()
2071    .chain(
2072        schedule
2073            .layers
2074            .iter()
2075            .map(|layer| vector_bytes(&layer.vertices)),
2076    )
2077    .chain(schedule.cycles.iter().map(vector_bytes))
2078    .sum()
2079}
2080
2081#[derive(Debug, Clone)]
2082struct CachedScheduleEntry {
2083    topology_epoch: u64,
2084    /// Authority `(store revision, rev.dyn)` the schedule was planned from
2085    /// (design §8.4; always 0 without `unified_authority`).
2086    authority_revision: (u64, u64),
2087    /// The request's vertex list as runs of consecutive ids (formula ids
2088    /// come in column runs, so a whole-workbook request is a few runs).
2089    candidate_vertices: VertexIdRuns,
2090    schedule: Arc<crate::engine::scheduler::Schedule>,
2091}
2092
2093/// A vertex list stored as `(first id, run length)` runs of consecutive
2094/// ids, in list order.
2095#[derive(Debug, Clone, Default)]
2096struct VertexIdRuns(Vec<(u32, u32)>);
2097
2098impl VertexIdRuns {
2099    fn from_slice(ids: &[VertexId]) -> Self {
2100        let mut runs: Vec<(u32, u32)> = Vec::new();
2101        for v in ids {
2102            match runs.last_mut() {
2103                Some((first, len)) if first.checked_add(*len) == Some(v.0) => *len += 1,
2104                _ => runs.push((v.0, 1)),
2105            }
2106        }
2107        runs.shrink_to_fit();
2108        Self(runs)
2109    }
2110
2111    fn equals(&self, ids: &[VertexId]) -> bool {
2112        let mut rest = ids;
2113        for &(first, len) in &self.0 {
2114            let len = len as usize;
2115            if rest.len() < len {
2116                return false;
2117            }
2118            let (head, tail) = rest.split_at(len);
2119            if head
2120                .iter()
2121                .enumerate()
2122                .any(|(i, v)| v.0 != first.wrapping_add(i as u32))
2123            {
2124                return false;
2125            }
2126            rest = tail;
2127        }
2128        rest.is_empty()
2129    }
2130
2131    fn heap_bytes(&self) -> usize {
2132        self.0.capacity() * std::mem::size_of::<(u32, u32)>()
2133    }
2134
2135    /// The number of ids.
2136    fn len(&self) -> usize {
2137        self.0.iter().map(|&(_, len)| len as usize).sum()
2138    }
2139}
2140
2141#[cfg(test)]
2142mod vertex_id_runs_tests {
2143    use super::{VertexId, VertexIdRuns};
2144
2145    #[test]
2146    fn runs_compare_like_the_list() {
2147        let ids = |v: &[u32]| v.iter().map(|&i| VertexId(i)).collect::<Vec<_>>();
2148        let list = ids(&[5, 6, 7, 2, 3, 9, 10, 10]);
2149        let runs = VertexIdRuns::from_slice(&list);
2150        assert_eq!(runs.0, vec![(5, 3), (2, 2), (9, 2), (10, 1)]);
2151        assert!(runs.equals(&list));
2152        assert!(!runs.equals(&list[..7]));
2153        assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 11])));
2154        assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 10, 11])));
2155        assert!(VertexIdRuns::from_slice(&[]).equals(&[]));
2156        assert!(!VertexIdRuns::from_slice(&[]).equals(&list));
2157        let edge = ids(&[u32::MAX - 1, u32::MAX, 0]);
2158        assert!(VertexIdRuns::from_slice(&edge).equals(&edge));
2159    }
2160}
2161
2162/// Uncacheable requests keep their schedule inline without a shared allocation.
2163enum EvaluationSchedule {
2164    Owned(crate::engine::scheduler::Schedule),
2165    Shared(Arc<crate::engine::scheduler::Schedule>),
2166}
2167
2168impl std::ops::Deref for EvaluationSchedule {
2169    type Target = crate::engine::scheduler::Schedule;
2170
2171    fn deref(&self) -> &Self::Target {
2172        match self {
2173            Self::Owned(schedule) => schedule,
2174            Self::Shared(schedule) => schedule,
2175        }
2176    }
2177}
2178
2179type ScheduleBuildOutput = (
2180    crate::engine::scheduler::Schedule,
2181    FxHashMap<VertexId, Vec<VertexId>>,
2182    ScheduleBuildMeta,
2183);
2184
2185type EvaluationScheduleBuildOutput = (
2186    EvaluationSchedule,
2187    FxHashMap<VertexId, Vec<VertexId>>,
2188    ScheduleBuildMeta,
2189);
2190
2191/// Opaque, revision-bound recalculation recipe.
2192#[derive(Debug)]
2193pub struct RecalcPlan {
2194    key: RecalcPlanKey,
2195    kind: RecalcPlanKind,
2196}
2197
2198#[derive(Debug)]
2199struct RecalcPlanKey {
2200    engine_token: Arc<()>,
2201    revisions: PlanningRevisionSnapshot,
2202}
2203
2204#[derive(Clone, Debug, PartialEq, Eq)]
2205struct PlanningRevisionSnapshot {
2206    engine_topology_epoch: u64,
2207    graph_topology_revision: u64,
2208    staged: u64,
2209    symbols: u64,
2210    semantic: u64,
2211    provider: Option<u64>,
2212    deterministic_mode: crate::engine::DeterministicMode,
2213    budgets: crate::engine::EvaluationBudgets,
2214}
2215
2216#[derive(Debug)]
2217enum RecalcPlanKind {
2218    CompatibilityFull {
2219        schedule: crate::engine::Schedule,
2220        has_dynamic_refs: bool,
2221    },
2222    Target {
2223        targets: Vec<crate::engine::EvaluationTarget>,
2224        scope: crate::engine::PrepareScope,
2225        topology: RecalcTopology,
2226        dynamic_policy: DynamicPlanPolicy,
2227    },
2228}
2229
2230#[derive(Debug)]
2231enum RecalcTopology {
2232    RunLocalRecipe,
2233    Workbook,
2234}
2235
2236#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2237enum DynamicPlanPolicy {
2238    BoundedTargetReplan,
2239}
2240
2241impl RecalcPlan {
2242    /// Returns the retained compatibility schedule depth. Target plans retain a
2243    /// run-local recipe rather than a schedule, so their layer count is zero.
2244    pub fn layer_count(&self) -> usize {
2245        match &self.kind {
2246            RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2247            RecalcPlanKind::Target { .. } => 0,
2248        }
2249    }
2250
2251    pub fn has_dynamic_refs(&self) -> bool {
2252        match &self.kind {
2253            RecalcPlanKind::CompatibilityFull {
2254                has_dynamic_refs, ..
2255            } => *has_dynamic_refs,
2256            RecalcPlanKind::Target { .. } => false,
2257        }
2258    }
2259
2260    #[cfg(test)]
2261    pub(crate) fn force_stale_reasons_for_test(
2262        &mut self,
2263        reasons: &[formualizer_common::PlanStaleReason],
2264    ) {
2265        use formualizer_common::PlanStaleReason;
2266        for reason in reasons {
2267            match reason {
2268                PlanStaleReason::Engine => {
2269                    self.key.engine_token = Arc::new(());
2270                }
2271                PlanStaleReason::Provider => {
2272                    self.key.revisions.provider = Some(
2273                        self.key
2274                            .revisions
2275                            .provider
2276                            .unwrap_or_default()
2277                            .wrapping_add(1),
2278                    );
2279                }
2280                PlanStaleReason::Semantic => {
2281                    self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2282                }
2283                PlanStaleReason::Budget => {
2284                    let current = self.key.revisions.budgets.work.max_work_units;
2285                    self.key.revisions.budgets.work.max_work_units =
2286                        Some(current.unwrap_or_default().wrapping_add(1));
2287                }
2288                PlanStaleReason::Staged => {
2289                    self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2290                }
2291                PlanStaleReason::Symbols => {
2292                    self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2293                }
2294                PlanStaleReason::Graph => {
2295                    self.key.revisions.graph_topology_revision =
2296                        self.key.revisions.graph_topology_revision.wrapping_add(1);
2297                }
2298                _ => {}
2299            }
2300        }
2301    }
2302}
2303
2304#[cfg(any(test, feature = "test-support"))]
2305pub(crate) mod criteria_mask_test_hooks {
2306    use std::cell::Cell;
2307
2308    thread_local! {
2309        static MASK_CALLS_ROWS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
2310        static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2311        static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2312    }
2313
2314    pub(crate) fn take_mask_work() -> (usize, usize) {
2315        MASK_CALLS_ROWS.with(|c| c.replace((0, 0)))
2316    }
2317
2318    pub(crate) fn note_mask(rows: usize) {
2319        MASK_CALLS_ROWS.with(|c| {
2320            let (calls, work) = c.get();
2321            c.set((calls + 1, work + rows));
2322        });
2323    }
2324
2325    pub fn reset_text_segment_counters() {
2326        TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2327        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2328    }
2329
2330    pub fn text_segment_counters() -> (usize, usize) {
2331        let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2332        let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2333        (a, b)
2334    }
2335
2336    pub(crate) fn inc_total() {
2337        TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2338    }
2339    pub(crate) fn inc_all_null() {
2340        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2341    }
2342}
2343
2344#[cfg(test)]
2345pub(crate) mod visibility_mask_test_hooks {
2346    use std::cell::Cell;
2347
2348    thread_local! {
2349        static HITS: Cell<usize> = const { Cell::new(0) };
2350        static MISSES: Cell<usize> = const { Cell::new(0) };
2351        static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2352    }
2353
2354    pub fn reset() {
2355        HITS.with(|c| c.set(0));
2356        MISSES.with(|c| c.set(0));
2357        EVICTIONS.with(|c| c.set(0));
2358    }
2359
2360    pub fn counters() -> (usize, usize, usize) {
2361        let hits = HITS.with(|c| c.get());
2362        let misses = MISSES.with(|c| c.get());
2363        let evictions = EVICTIONS.with(|c| c.get());
2364        (hits, misses, evictions)
2365    }
2366
2367    pub(crate) fn inc_hit() {
2368        HITS.with(|c| c.set(c.get() + 1));
2369    }
2370
2371    pub(crate) fn inc_miss() {
2372        MISSES.with(|c| c.set(c.get() + 1));
2373    }
2374
2375    pub(crate) fn inc_eviction() {
2376        EVICTIONS.with(|c| c.set(c.get() + 1));
2377    }
2378}
2379
2380fn is_numeric_text_equality(pred: &crate::args::CriteriaPredicate) -> bool {
2381    match pred {
2382        crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(text)) => {
2383            text.trim().parse::<f64>().is_ok_and(f64::is_finite)
2384        }
2385        _ => false,
2386    }
2387}
2388
2389/// Resolve the null (non-number) rows of an `=n` / `<>n` criteria mask over
2390/// one row segment: numeric text compares by value, every other non-number
2391/// cell (blank, boolean, error) is unequal. Segments without text stay
2392/// vectorized.
2393fn fill_numeric_equality_nulls(
2394    view: &RangeView<'_>,
2395    row_start: usize,
2396    row_len: usize,
2397    col_in_view: usize,
2398    mask: &arrow_array::BooleanArray,
2399    n: f64,
2400    ne: bool,
2401) -> Option<arrow_array::BooleanArray> {
2402    use arrow_array::Array as _;
2403    let texts = view.slice_lowered_text(row_start, row_len);
2404    let text = texts.get(col_in_view).and_then(|t| t.as_ref());
2405    match text {
2406        None => {
2407            if !ne {
2408                // Nulls are already "no match".
2409                return Some(mask.clone());
2410            }
2411            let nulls = arrow::compute::is_null(mask).ok()?;
2412            crate::compute_prelude::boolean::or_kleene(mask, &nulls).ok()
2413        }
2414        Some(text) => {
2415            if text.len() != mask.len() {
2416                return None;
2417            }
2418            let mut out = arrow_array::builder::BooleanBuilder::with_capacity(mask.len());
2419            for i in 0..mask.len() {
2420                if mask.is_valid(i) {
2421                    out.append_value(mask.value(i));
2422                } else if text.is_valid(i) {
2423                    let equal = crate::locale::Locale::invariant()
2424                        .parse_number_invariant(text.value(i))
2425                        .is_some_and(|x| (x - n).abs() < 1e-12);
2426                    out.append_value(equal != ne);
2427                } else {
2428                    out.append_value(ne);
2429                }
2430            }
2431            Some(out.finish())
2432        }
2433    }
2434}
2435
2436fn compute_criteria_mask(
2437    view: &RangeView<'_>,
2438    col_in_view: usize,
2439    pred: &crate::args::CriteriaPredicate,
2440) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2441    use crate::compute_prelude::{boolean, cmp, concat_arrays};
2442    use arrow::compute::kernels::comparison::{ilike, nilike};
2443    use arrow_array::{
2444        Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2445    };
2446
2447    // Helper: apply a numeric predicate to a single Float64Array chunk
2448    fn apply_numeric_pred(
2449        chunk: &Float64Array,
2450        pred: &crate::args::CriteriaPredicate,
2451    ) -> Option<BooleanArray> {
2452        match pred {
2453            crate::args::CriteriaPredicate::Gt(n) => {
2454                cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2455            }
2456            crate::args::CriteriaPredicate::Ge(n) => {
2457                cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2458            }
2459            crate::args::CriteriaPredicate::Lt(n) => {
2460                cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2461            }
2462            crate::args::CriteriaPredicate::Le(n) => {
2463                cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2464            }
2465            crate::args::CriteriaPredicate::Eq(v) => match v {
2466                formualizer_common::LiteralValue::Number(x) => {
2467                    cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2468                }
2469                formualizer_common::LiteralValue::Int(i) => {
2470                    cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2471                }
2472                _ => None,
2473            },
2474            crate::args::CriteriaPredicate::Ne(v) => match v {
2475                formualizer_common::LiteralValue::Number(x) => {
2476                    cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2477                }
2478                formualizer_common::LiteralValue::Int(i) => {
2479                    cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2480                }
2481                _ => None,
2482            },
2483            _ => None,
2484        }
2485    }
2486
2487    // Check if this is a numeric predicate that can be applied per-chunk
2488    let is_numeric_pred = matches!(
2489        pred,
2490        crate::args::CriteriaPredicate::Gt(_)
2491            | crate::args::CriteriaPredicate::Ge(_)
2492            | crate::args::CriteriaPredicate::Lt(_)
2493            | crate::args::CriteriaPredicate::Le(_)
2494            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2495            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2496            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2497            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2498    );
2499
2500    // OPTIMIZED PATH: For numeric predicates, apply per-chunk and concatenate boolean masks.
2501    // This avoids materializing the full numeric column (64-bit per element) and instead
2502    // concatenates boolean masks (1-bit per element) - a 64x memory reduction.
2503    if is_numeric_pred {
2504        let mut bool_parts: Vec<BooleanArray> = Vec::new();
2505        // `=n` / `<>n`: the number lane is null for blank, text, boolean and
2506        // error cells. Blanks, booleans and errors never equal a number;
2507        // numeric text does (`5` matches "5"), as in `criteria_match`.
2508        let number = |v: &formualizer_common::LiteralValue| match v {
2509            formualizer_common::LiteralValue::Number(n) => Some(*n),
2510            formualizer_common::LiteralValue::Int(i) => Some(*i as f64),
2511            _ => None,
2512        };
2513        let equality = match pred {
2514            crate::args::CriteriaPredicate::Eq(v) => number(v).map(|n| (n, false)),
2515            crate::args::CriteriaPredicate::Ne(v) => number(v).map(|n| (n, true)),
2516            _ => None,
2517        };
2518        for res in view.numbers_slices() {
2519            let (rs, rl, cols_seg) = res.ok()?;
2520            if col_in_view < cols_seg.len() {
2521                let chunk = cols_seg[col_in_view].as_ref();
2522                let mut mask = apply_numeric_pred(chunk, pred)?;
2523                if let Some((n, ne)) = equality
2524                    && mask.null_count() > 0
2525                {
2526                    mask = fill_numeric_equality_nulls(view, rs, rl, col_in_view, &mask, n, ne)?;
2527                }
2528                bool_parts.push(mask);
2529            }
2530        }
2531
2532        if bool_parts.is_empty() {
2533            return None;
2534        } else if bool_parts.len() == 1 {
2535            return Some(std::sync::Arc::new(bool_parts.remove(0)));
2536        } else {
2537            // Concatenate boolean masks (much cheaper than concatenating Float64 arrays)
2538            let anys: Vec<&dyn arrow_array::Array> = bool_parts
2539                .iter()
2540                .map(|a| a as &dyn arrow_array::Array)
2541                .collect();
2542            let conc: ArrayRef = concat_arrays(&anys).ok()?;
2543            let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2544            return Some(std::sync::Arc::new(ba));
2545        }
2546    }
2547
2548    // Wildcards and numeric text equality can match non-text cells. The lowered
2549    // base lane is text-only, unlike the scalar matcher. Keep the vectorized
2550    // path for text-only data, but cache a scalar-equivalent mask for mixed data.
2551    if is_numeric_text_equality(pred)
2552        || matches!(pred, crate::args::CriteriaPredicate::TextLike { .. })
2553    {
2554        for tags in view.type_tags_slices() {
2555            let (_, _, cols) = tags.ok()?;
2556            let tags = cols.get(col_in_view)?;
2557            if tags.values().iter().any(|tag| {
2558                *tag == crate::arrow_store::TypeTag::Empty as u8
2559                    || *tag == crate::arrow_store::TypeTag::Number as u8
2560                    || *tag == crate::arrow_store::TypeTag::Boolean as u8
2561            }) {
2562                let mut mask = BooleanBuilder::new();
2563                for chunk in view.iter_row_chunks() {
2564                    let chunk = chunk.ok()?;
2565                    for row in chunk.row_start..chunk.row_start + chunk.row_len {
2566                        mask.append_value(crate::builtins::criteria_match(
2567                            pred,
2568                            &view.get_cell(row, col_in_view),
2569                        ));
2570                    }
2571                }
2572                return Some(std::sync::Arc::new(mask.finish()));
2573            }
2574        }
2575    }
2576
2577    // Keep spreadsheet tilde escapes in the shared scalar matcher.
2578    if matches!(pred, crate::args::CriteriaPredicate::TextLike { pattern, .. }
2579        if pattern.contains('~'))
2580    {
2581        return None;
2582    }
2583
2584    // TEXT PATH: build masks per row-chunk using lowered text slices.
2585    // This avoids concatenating full-string columns just to compute a boolean mask.
2586    // `"="` / `"<>"`: truly blank cells only (empty text is not blank).
2587    if matches!(
2588        pred,
2589        crate::args::CriteriaPredicate::IsBlank | crate::args::CriteriaPredicate::NotBlank
2590    ) {
2591        let want_blank = matches!(pred, crate::args::CriteriaPredicate::IsBlank);
2592        let mut bool_parts: Vec<BooleanArray> = Vec::new();
2593        for tags in view.type_tags_slices() {
2594            let (_, _, cols) = tags.ok()?;
2595            let tags = cols.get(col_in_view)?;
2596            let mut bb = BooleanBuilder::with_capacity(tags.len());
2597            for i in 0..tags.len() {
2598                let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8;
2599                bb.append_value(blank == want_blank);
2600            }
2601            bool_parts.push(bb.finish());
2602        }
2603        return match bool_parts.len() {
2604            0 => None,
2605            1 => Some(std::sync::Arc::new(bool_parts.remove(0))),
2606            _ => {
2607                let anys: Vec<&dyn arrow_array::Array> = bool_parts
2608                    .iter()
2609                    .map(|a| a as &dyn arrow_array::Array)
2610                    .collect();
2611                let conc: ArrayRef = concat_arrays(&anys).ok()?;
2612                let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2613                Some(std::sync::Arc::new(ba))
2614            }
2615        };
2616    }
2617
2618    // Arrow LIKE uses backslash escapes; SQL punctuation is literal in Excel.
2619    let like_pattern = |text: &str, wildcards: bool| {
2620        let mut pattern = String::with_capacity(text.len());
2621        for ch in text.chars() {
2622            match ch {
2623                '%' | '_' | '\\' => {
2624                    pattern.push('\\');
2625                    pattern.push(ch);
2626                }
2627                '*' if wildcards => pattern.push('%'),
2628                '?' if wildcards => pattern.push('_'),
2629                _ => pattern.push(ch),
2630            }
2631        }
2632        pattern
2633    };
2634    let (text_kind, text_pat, empty_special) = match pred {
2635        crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2636            (0u8, like_pattern(&t.to_lowercase(), false), t.is_empty())
2637        }
2638        crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2639            (1u8, like_pattern(&t.to_lowercase(), false), false)
2640        }
2641        crate::args::CriteriaPredicate::TextLike {
2642            pattern,
2643            case_insensitive,
2644        } => {
2645            let p = if *case_insensitive {
2646                pattern.to_lowercase()
2647            } else {
2648                pattern.clone()
2649            };
2650            (2u8, like_pattern(&p, true), false)
2651        }
2652        _ => return None,
2653    };
2654
2655    let text_pat_is_empty = text_pat.is_empty();
2656    let ne_matches_blank = text_kind == 1 && !text_pat_is_empty;
2657    let pat = StringArray::new_scalar(text_pat);
2658    let mut bool_parts: Vec<BooleanArray> = Vec::new();
2659
2660    let mut tag_slices = view.type_tags_slices();
2661    for res in view.iter_row_chunks() {
2662        let cs = res.ok()?;
2663        if cs.row_len == 0 {
2664            continue;
2665        }
2666        #[cfg(test)]
2667        criteria_mask_test_hooks::inc_total();
2668
2669        let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2670        if col_in_view >= slices.len() {
2671            return None;
2672        }
2673
2674        let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2675        if empty_special || (text_kind == 1 && text_pat_is_empty) {
2676            let (tag_start, tag_len, tags) = tag_slices.next()?.ok()?;
2677            if tag_start != cs.row_start || tag_len != cs.row_len {
2678                return None;
2679            }
2680            let tags = tags.get(col_in_view)?;
2681            // A null text lane is not a blank cell: base numeric/boolean/error
2682            // cells also have null text. Consult the overlay-aware type tags,
2683            // and inspect strings only to distinguish empty text from text.
2684            let strings = seg_opt.and_then(|a| a.as_any().downcast_ref::<StringArray>());
2685            let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2686            for i in 0..cs.row_len {
2687                let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8
2688                    || (tags.value(i) == crate::arrow_store::TypeTag::Text as u8
2689                        && strings.is_some_and(|s| s.is_valid(i) && s.value(i).is_empty()));
2690                bb.append_value(if text_kind == 0 { blank } else { !blank });
2691            }
2692            #[cfg(test)]
2693            if seg_opt.is_none() {
2694                criteria_mask_test_hooks::inc_all_null();
2695            }
2696            bool_parts.push(bb.finish());
2697            continue;
2698        }
2699        let seg = match seg_opt {
2700            Some(s) => s,
2701            None => {
2702                #[cfg(test)]
2703                criteria_mask_test_hooks::inc_all_null();
2704                if (text_kind == 0 && empty_special) || ne_matches_blank {
2705                    // Eq("") treats nulls (Empty) as equal.
2706                    let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2707                    bb.append_n(cs.row_len, true);
2708                    bool_parts.push(bb.finish());
2709                } else {
2710                    // For non-empty patterns, ilike/nilike return null on null inputs.
2711                    bool_parts.push(BooleanArray::new_null(cs.row_len));
2712                }
2713                continue;
2714            }
2715        };
2716
2717        let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2718        let mut m = match text_kind {
2719            0 => ilike(seg_sa, &pat).ok()?,
2720            1 => nilike(seg_sa, &pat).ok()?,
2721            2 => ilike(seg_sa, &pat).ok()?,
2722            _ => return None,
2723        };
2724
2725        // Only fold blank/Empty (null) cells into the mask when the segment
2726        // actually contains any. The null-fill loop + or_kleene are pure
2727        // overhead on blank-free chunks, so a `<>text` (or `=""`) aggregation
2728        // over a column with no blanks stays fully vectorized on the ilike/
2729        // nilike result.
2730        if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2731            // Treat nulls as equal to empty string
2732            let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2733            for i in 0..seg_sa.len() {
2734                bb.append_value(seg_sa.is_null(i));
2735            }
2736            let nulls = bb.finish();
2737            m = boolean::or_kleene(&m, &nulls).ok()?;
2738        }
2739
2740        bool_parts.push(m);
2741    }
2742
2743    if bool_parts.is_empty() {
2744        None
2745    } else if bool_parts.len() == 1 {
2746        Some(std::sync::Arc::new(bool_parts.remove(0)))
2747    } else {
2748        let anys: Vec<&dyn arrow_array::Array> = bool_parts
2749            .iter()
2750            .map(|a| a as &dyn arrow_array::Array)
2751            .collect();
2752        let conc: ArrayRef = concat_arrays(&anys).ok()?;
2753        let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2754        Some(std::sync::Arc::new(ba))
2755    }
2756}
2757
2758#[derive(Debug, Clone)]
2759pub struct LayerInfo {
2760    pub vertex_count: usize,
2761    pub parallel_eligible: bool,
2762    pub sample_cells: Vec<String>, // Sample of up to 5 cell addresses
2763}
2764
2765#[derive(Debug, Clone)]
2766pub struct EvalPlan {
2767    pub total_vertices_to_evaluate: usize,
2768    pub layers: Vec<LayerInfo>,
2769    pub cycles_detected: usize,
2770    pub dirty_count: usize,
2771    pub volatile_count: usize,
2772    pub parallel_enabled: bool,
2773    pub estimated_parallel_layers: usize,
2774    pub target_cells: Vec<String>,
2775}
2776
2777/// Whether the configured FormulaPlane mode is ignored (spans never placed):
2778/// always, since the dependency authority is the runtime path (design §10).
2779#[inline]
2780fn plane_mode_ignored() -> bool {
2781    true
2782}
2783
2784/// Test-support probe for sibling-crate tests: true when span placement is
2785/// off because the FormulaPlane mode is ignored (see `plane_mode_ignored`).
2786#[cfg(feature = "test-support")]
2787#[doc(hidden)]
2788pub fn formula_plane_mode_ignored_for_test() -> bool {
2789    plane_mode_ignored()
2790}
2791
2792impl<R> Engine<R>
2793where
2794    R: EvaluationContext,
2795{
2796    /// # Panics
2797    /// Panics when `config.cycle` is invalid ([`CycleConfig::validate`],
2798    /// spec §2): `Iterate` with `detection: Static`, `max_iterations == 0`,
2799    /// or a negative/non-finite `max_change`. `EvalConfig::with_cycle`
2800    /// rejects these at build; this re-validates configs assembled via
2801    /// struct literals.
2802    pub fn new(resolver: R, config: EvalConfig) -> Self {
2803        // Under the unified authority the FormulaPlane mode is accepted and
2804        // ignored (design §10). Normalizing the stored mode keeps external
2805        // readers of `config` (loaders choosing a span-preparation route) on
2806        // the per-cell path; engine reads go through `formula_plane_mode()`.
2807        let config = if plane_mode_ignored() {
2808            EvalConfig {
2809                formula_plane_mode: FormulaPlaneMode::Off,
2810                ..config
2811            }
2812        } else {
2813            config
2814        };
2815        if let Err(msg) = config.cycle.validate() {
2816            panic!("invalid CycleConfig: {msg}");
2817        }
2818        crate::builtins::load_builtins();
2819        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2820            &config.evaluation_budgets,
2821            config.max_vertices,
2822            config.max_memory_mb,
2823            config.max_eval_time,
2824        );
2825
2826        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2827            #[cfg(feature = "system-clock")]
2828            {
2829                Arc::new(crate::timezone::SystemClock::new(
2830                    crate::timezone::TimeZoneSpec::default(),
2831                ))
2832            }
2833            #[cfg(not(feature = "system-clock"))]
2834            {
2835                Arc::new(crate::timezone::FixedClock::new(
2836                    chrono::DateTime::UNIX_EPOCH,
2837                    crate::timezone::TimeZoneSpec::Utc,
2838                ))
2839            }
2840        });
2841
2842        // Initialize thread pool based on config
2843        let thread_pool = if config.enable_parallel {
2844            let mut builder = ThreadPoolBuilder::new();
2845            if let Some(max_threads) = config.max_threads {
2846                builder = builder.num_threads(max_threads);
2847            }
2848
2849            match builder.build() {
2850                Ok(pool) => Some(Arc::new(pool)),
2851                Err(_) => {
2852                    // Fall back to sequential evaluation if thread pool creation fails
2853                    None
2854                }
2855            }
2856        } else {
2857            None
2858        };
2859
2860        // C1a retained/cache budgets are observational; cache defaults stay explicit.
2861        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2862        let function_provider_revision_seen = resolver.planning_semantic_revision();
2863        let mut engine = Self {
2864            graph: DependencyGraph::new_with_config(config.clone()),
2865            resolver,
2866            config,
2867            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2868            clock: crate::timezone::SnapshotClock::new(clock),
2869            thread_pool,
2870            recalc_epoch: 0,
2871            snapshot_id: std::sync::atomic::AtomicU64::new(1),
2872            topology_epoch: 0,
2873            cached_static_schedule: None,
2874            recent_schedules: Vec::new(),
2875            base_schedule: None,
2876            #[cfg(any(test, feature = "benchmark_internal"))]
2877            recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
2878            spill_mgr: ShimSpillManager::default(),
2879            arrow_sheets: SheetStore::default(),
2880            format_registry: crate::format::FormatRegistry::default(),
2881            derived_formats: Default::default(),
2882            #[cfg(test)]
2883            derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
2884            #[cfg(test)]
2885            family_members_for_test: std::sync::atomic::AtomicU64::new(0),
2886            #[cfg(test)]
2887            invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
2888            #[cfg(test)]
2889            lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
2890            #[cfg(test)]
2891            chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
2892            #[cfg(test)]
2893            lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
2894            #[cfg(test)]
2895            criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
2896            #[cfg(test)]
2897            memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
2898            compressed_at_build: None,
2899            #[cfg(test)]
2900            computed_overlay_set_explicit_entry_operations_for_test: 0,
2901            #[cfg(test)]
2902            computed_overlay_stale_clear_range_effects_for_test: 0,
2903            #[cfg(test)]
2904            computed_overlay_stale_clear_offset_attempts_for_test: 0,
2905            #[cfg(test)]
2906            computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
2907            has_edited: false,
2908            overlay_compactions: 0,
2909            computed_overlay_bytes_estimate: 0,
2910            computed_overlay_mirroring_disabled: false,
2911            force_materialize_range_views: false,
2912            row_bounds_cache: std::sync::RwLock::new(None),
2913            used_axis_bounds_cache: std::sync::RwLock::new(None),
2914            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2915            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2916            source_formula_token: Arc::new(()),
2917            recalc_plan_token: Arc::new(()),
2918            staged_formulas: std::collections::HashMap::new(),
2919            staged_formula_index: StagedFormulaIndex::default(),
2920            blocked_pending_spills: Vec::new(),
2921            spill_blocker_waiters: SpillBlockerWaiters::default(),
2922            row_visibility: FxHashMap::default(),
2923            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2924            formula_parse_diagnostics: Vec::new(),
2925            last_formula_ingest_report: None,
2926            formula_ingest_report_total: FormulaIngestReport::default(),
2927            active_cancel_flag: None,
2928            active_evaluation_deadline: None,
2929            action_depth: 0,
2930            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2931            virtual_dep_fallback_activations: 0,
2932            last_cycle_telemetry: CycleTelemetry::default(),
2933            next_evaluation_resource_request_id: 1,
2934            evaluation_resource_request_depth: 0,
2935            active_evaluation_resource_request: None,
2936            last_evaluation_resource_request: None,
2937            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2938            evaluation_resource_request_started_at: None,
2939            evaluation_resource_budgets: resolved_resources.budgets,
2940            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2941            active_resource_ledger: None,
2942            source_cache_footprints: Vec::new(),
2943            source_cache_accounted: 0,
2944            pending_iterative_redirty: Vec::new(),
2945            snapshot_evaluation_active: false,
2946            snapshot_volatile_redirty_pending: false,
2947            retained_scc_members: FxHashMap::default(),
2948            next_retained_scc_id: 0,
2949            retained_scc_config_fingerprint: 0,
2950            retained_scc_function_epoch_seen: 0,
2951            retained_scc_provider_revision_seen: None,
2952            retained_scc_dirty_at_begin: Vec::new(),
2953            iterative_state_values: FxHashMap::default(),
2954            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2955            function_provider_revision_seen,
2956            #[cfg(feature = "tracing")]
2957            trace_evaluation_counters: TraceEvaluationCounters::default(),
2958            #[cfg(test)]
2959            evaluation_request_begin_count_for_test: 0,
2960            #[cfg(any(test, feature = "test-support"))]
2961            before_prepared_span_commit_hook: None,
2962            #[cfg(test)]
2963            before_target_preparation_commit_hook: None,
2964            #[cfg(test)]
2965            before_target_planning_snapshot_hook: None,
2966            #[cfg(test)]
2967            inject_target_semantic_stale_once_for_test: false,
2968            #[cfg(test)]
2969            force_virtual_dep_changes_remaining_for_test: 0,
2970            freshness: Default::default(),
2971            #[cfg(test)]
2972            fail_evaluation_commit_preflight_once_for_test: false,
2973            #[cfg(test)]
2974            target_preparation_fault_for_test: None,
2975            #[cfg(test)]
2976            force_non_cycle_schedule_fallback_for_test: false,
2977            #[cfg(test)]
2978            before_legacy_fallback_final_provider_sample_hook: None,
2979            #[cfg(test)]
2980            after_eager_proposal_commit_hook: None,
2981        };
2982        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
2983        engine.config.arrow_storage_enabled = true;
2984        engine.config.delta_overlay_enabled = true;
2985        engine.config.write_formula_overlay_enabled = true;
2986        let default_sheet = engine.graph.default_sheet_name().to_string();
2987        engine.ensure_arrow_sheet(&default_sheet);
2988        engine
2989    }
2990
2991    /// Create an Engine with a custom thread pool (for shared thread pool scenarios)
2992    ///
2993    /// # Panics
2994    /// Panics when `config.cycle` is invalid, exactly like [`Engine::new`].
2995    pub fn with_thread_pool(
2996        resolver: R,
2997        config: EvalConfig,
2998        thread_pool: Arc<rayon::ThreadPool>,
2999    ) -> Self {
3000        if let Err(msg) = config.cycle.validate() {
3001            panic!("invalid CycleConfig: {msg}");
3002        }
3003        crate::builtins::load_builtins();
3004        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
3005            &config.evaluation_budgets,
3006            config.max_vertices,
3007            config.max_memory_mb,
3008            config.max_eval_time,
3009        );
3010        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
3011            #[cfg(feature = "system-clock")]
3012            {
3013                Arc::new(crate::timezone::SystemClock::new(
3014                    crate::timezone::TimeZoneSpec::default(),
3015                ))
3016            }
3017            #[cfg(not(feature = "system-clock"))]
3018            {
3019                Arc::new(crate::timezone::FixedClock::new(
3020                    chrono::DateTime::UNIX_EPOCH,
3021                    crate::timezone::TimeZoneSpec::Utc,
3022                ))
3023            }
3024        });
3025        // C1a retained/cache budgets are observational; cache defaults stay explicit.
3026        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
3027        let function_provider_revision_seen = resolver.planning_semantic_revision();
3028        let mut engine = Self {
3029            graph: DependencyGraph::new_with_config(config.clone()),
3030            resolver,
3031            config,
3032            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
3033            clock: crate::timezone::SnapshotClock::new(clock),
3034            thread_pool: Some(thread_pool),
3035            recalc_epoch: 0,
3036            snapshot_id: std::sync::atomic::AtomicU64::new(1),
3037            topology_epoch: 0,
3038            cached_static_schedule: None,
3039            recent_schedules: Vec::new(),
3040            base_schedule: None,
3041            #[cfg(any(test, feature = "benchmark_internal"))]
3042            recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
3043            spill_mgr: ShimSpillManager::default(),
3044            arrow_sheets: SheetStore::default(),
3045            format_registry: crate::format::FormatRegistry::default(),
3046            derived_formats: Default::default(),
3047            #[cfg(test)]
3048            derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
3049            #[cfg(test)]
3050            family_members_for_test: std::sync::atomic::AtomicU64::new(0),
3051            #[cfg(test)]
3052            invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
3053            #[cfg(test)]
3054            lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
3055            #[cfg(test)]
3056            chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
3057            #[cfg(test)]
3058            lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
3059            #[cfg(test)]
3060            criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
3061            #[cfg(test)]
3062            memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
3063            compressed_at_build: None,
3064            #[cfg(test)]
3065            computed_overlay_set_explicit_entry_operations_for_test: 0,
3066            #[cfg(test)]
3067            computed_overlay_stale_clear_range_effects_for_test: 0,
3068            #[cfg(test)]
3069            computed_overlay_stale_clear_offset_attempts_for_test: 0,
3070            #[cfg(test)]
3071            computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
3072            has_edited: false,
3073            overlay_compactions: 0,
3074            computed_overlay_bytes_estimate: 0,
3075            computed_overlay_mirroring_disabled: false,
3076            force_materialize_range_views: false,
3077            row_bounds_cache: std::sync::RwLock::new(None),
3078            used_axis_bounds_cache: std::sync::RwLock::new(None),
3079            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
3080            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
3081            source_formula_token: Arc::new(()),
3082            recalc_plan_token: Arc::new(()),
3083            staged_formulas: std::collections::HashMap::new(),
3084            staged_formula_index: StagedFormulaIndex::default(),
3085            blocked_pending_spills: Vec::new(),
3086            spill_blocker_waiters: SpillBlockerWaiters::default(),
3087            row_visibility: FxHashMap::default(),
3088            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
3089            formula_parse_diagnostics: Vec::new(),
3090            last_formula_ingest_report: None,
3091            formula_ingest_report_total: FormulaIngestReport::default(),
3092            active_cancel_flag: None,
3093            active_evaluation_deadline: None,
3094            action_depth: 0,
3095            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
3096            virtual_dep_fallback_activations: 0,
3097            last_cycle_telemetry: CycleTelemetry::default(),
3098            next_evaluation_resource_request_id: 1,
3099            evaluation_resource_request_depth: 0,
3100            active_evaluation_resource_request: None,
3101            last_evaluation_resource_request: None,
3102            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
3103            evaluation_resource_request_started_at: None,
3104            evaluation_resource_budgets: resolved_resources.budgets,
3105            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
3106            active_resource_ledger: None,
3107            source_cache_footprints: Vec::new(),
3108            source_cache_accounted: 0,
3109            pending_iterative_redirty: Vec::new(),
3110            snapshot_evaluation_active: false,
3111            snapshot_volatile_redirty_pending: false,
3112            retained_scc_members: FxHashMap::default(),
3113            next_retained_scc_id: 0,
3114            retained_scc_config_fingerprint: 0,
3115            retained_scc_function_epoch_seen: 0,
3116            retained_scc_provider_revision_seen: None,
3117            retained_scc_dirty_at_begin: Vec::new(),
3118            iterative_state_values: FxHashMap::default(),
3119            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
3120            function_provider_revision_seen,
3121            #[cfg(feature = "tracing")]
3122            trace_evaluation_counters: TraceEvaluationCounters::default(),
3123            #[cfg(test)]
3124            evaluation_request_begin_count_for_test: 0,
3125            #[cfg(any(test, feature = "test-support"))]
3126            before_prepared_span_commit_hook: None,
3127            #[cfg(test)]
3128            before_target_preparation_commit_hook: None,
3129            #[cfg(test)]
3130            before_target_planning_snapshot_hook: None,
3131            #[cfg(test)]
3132            inject_target_semantic_stale_once_for_test: false,
3133            #[cfg(test)]
3134            force_virtual_dep_changes_remaining_for_test: 0,
3135            freshness: Default::default(),
3136            #[cfg(test)]
3137            fail_evaluation_commit_preflight_once_for_test: false,
3138            #[cfg(test)]
3139            target_preparation_fault_for_test: None,
3140            #[cfg(test)]
3141            force_non_cycle_schedule_fallback_for_test: false,
3142            #[cfg(test)]
3143            before_legacy_fallback_final_provider_sample_hook: None,
3144            #[cfg(test)]
3145            after_eager_proposal_commit_hook: None,
3146        };
3147        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
3148        engine.config.arrow_storage_enabled = true;
3149        engine.config.delta_overlay_enabled = true;
3150        engine.config.write_formula_overlay_enabled = true;
3151        let default_sheet = engine.graph.default_sheet_name().to_string();
3152        engine.ensure_arrow_sheet(&default_sheet);
3153        engine
3154    }
3155
3156    pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
3157        &self.workbook_load_limits
3158    }
3159
3160    pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
3161        self.workbook_load_limits = limits;
3162    }
3163
3164    fn clear_source_cache(&self) {
3165        if let Ok(mut g) = self.source_cache.write() {
3166            *g = SourceCache::default();
3167        }
3168    }
3169
3170    pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
3171        &self.last_virtual_dep_telemetry
3172    }
3173
3174    /// Telemetry from Runtime SCC evaluation during the most recent
3175    /// evaluation request (always default-zero under `CycleDetection::Static`
3176    /// or when `enable_virtual_dep_telemetry` is off).
3177    pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
3178        &self.last_cycle_telemetry
3179    }
3180
3181    /// Resource observations for the most recently completed public evaluation request.
3182    pub fn last_evaluation_resource_request_stats(
3183        &self,
3184    ) -> Option<&EvaluationResourceRequestStats> {
3185        self.last_evaluation_resource_request.as_ref()
3186    }
3187
3188    /// Cumulative resource observations since engine creation or the last telemetry reset.
3189    pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
3190        self.evaluation_resource_baseline
3191    }
3192
3193    pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
3194        &self.evaluation_resource_budgets
3195    }
3196
3197    /// At most one diagnostic is emitted for deprecated resource fields.
3198    pub fn evaluation_resource_config_diagnostic(
3199        &self,
3200    ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
3201        self.evaluation_resource_config_diagnostic.as_ref()
3202    }
3203
3204    /// Reset accumulated and last-request observations without reusing request IDs.
3205    pub fn reset_evaluation_resource_telemetry(&mut self) {
3206        self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
3207        self.last_evaluation_resource_request = None;
3208    }
3209
3210    // Reconcile without replay locks: several packages may share the same Arc.
3211    // Weak tokens neither retain dead packages nor duplicate their allocations.
3212    fn reconcile_source_cache_footprints(&mut self) -> Result<(), ExcelError> {
3213        self.blocked_pending_spills.retain(|&(vertex, anchor, _)| {
3214            self.graph.vertex_exists(vertex)
3215                && self.graph.get_cell_ref(vertex) == Some(anchor)
3216                && matches!(
3217                    self.graph.get_vertex_kind(vertex),
3218                    VertexKind::FormulaScalar | VertexKind::FormulaArray
3219                )
3220        });
3221        if self.blocked_pending_spills.is_empty() {
3222            self.blocked_pending_spills = Vec::new();
3223        }
3224        let mut bytes = (self.blocked_pending_spills.capacity()
3225            * std::mem::size_of::<(VertexId, CellRef, Region)>()) as u64;
3226        self.source_cache_footprints.retain(|weak| {
3227            let Some(footprint) = weak.upgrade() else {
3228                return false;
3229            };
3230            bytes = bytes.saturating_add(footprint.load(std::sync::atomic::Ordering::Acquire));
3231            true
3232        });
3233        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3234            ledger
3235                .release_retained(self.source_cache_accounted)
3236                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3237            // Observe first: even a tightened budget cannot erase live ownership.
3238            ledger.observe_retained(bytes);
3239            self.source_cache_accounted = bytes;
3240            ledger
3241                .reserve_retained(0)
3242                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3243        }
3244        Ok(())
3245    }
3246
3247    fn duration_ns(duration: std::time::Duration) -> u64 {
3248        u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
3249    }
3250
3251    fn observe_evaluation_resource_request<T>(
3252        &mut self,
3253        kind: EvaluationRequestKind,
3254        evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3255    ) -> Result<T, ExcelError> {
3256        let outermost = self.evaluation_resource_request_depth == 0;
3257        if outermost && self.snapshot_volatile_redirty_pending {
3258            self.snapshot_volatile_redirty_pending = false;
3259            self.graph.redirty_volatiles();
3260        }
3261        #[cfg(feature = "tracing")]
3262        if outermost {
3263            self.trace_evaluation_counters = TraceEvaluationCounters::default();
3264        }
3265        #[cfg(feature = "tracing")]
3266        let request_kind = if matches!(
3267            kind,
3268            EvaluationRequestKind::Full
3269                | EvaluationRequestKind::FullWithDelta
3270                | EvaluationRequestKind::FullCancellable
3271                | EvaluationRequestKind::FullLogged
3272        ) {
3273            "full"
3274        } else {
3275            "targeted"
3276        };
3277        #[cfg(feature = "tracing")]
3278        let _request_span = outermost.then(|| {
3279            crate::engine::trace::fz_span!(
3280                tracing::Level::INFO,
3281                "evaluate",
3282                "evaluate.request",
3283                kind = request_kind,
3284                mode = ?FormulaPlaneMode::Off
3285            )
3286        });
3287        if outermost {
3288            let request_id = self.next_evaluation_resource_request_id;
3289            self.next_evaluation_resource_request_id = request_id
3290                .checked_add(1)
3291                .expect("evaluation resource request ID exhausted");
3292            self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
3293                request_id,
3294                kind,
3295                FormulaPlaneMode::Off,
3296                self.staged_formula_count(),
3297            ));
3298            self.evaluation_resource_baseline.record_started(request_id);
3299            self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
3300            self.source_cache_accounted = 0;
3301            self.active_resource_ledger = Some(ResourceLedger::new(
3302                Some(request_id),
3303                self.evaluation_resource_budgets.clone(),
3304            ));
3305        }
3306        self.evaluation_resource_request_depth =
3307            self.evaluation_resource_request_depth.saturating_add(1);
3308        let result = if outermost {
3309            self.reconcile_source_cache_footprints()
3310                .and_then(|()| self.resource_checkpoint(0))
3311                .and_then(|()| evaluate(self))
3312        } else {
3313            evaluate(self)
3314        };
3315        if outermost && result.is_err() {
3316            self.freshness_abort_pass();
3317        }
3318        self.evaluation_resource_request_depth =
3319            self.evaluation_resource_request_depth.saturating_sub(1);
3320
3321        let reconciliation = if outermost {
3322            self.reconcile_source_cache_footprints()
3323        } else {
3324            Ok(())
3325        };
3326        let result = result.and_then(|value| reconciliation.map(|()| value));
3327        if outermost {
3328            let total_ns = self
3329                .evaluation_resource_request_started_at
3330                .take()
3331                .map(|start| Self::duration_ns(start.elapsed()))
3332                .unwrap_or(0);
3333            let mut stats = self
3334                .active_evaluation_resource_request
3335                .take()
3336                .expect("outer evaluation resource request has active stats");
3337            let mut ledger = self
3338                .active_resource_ledger
3339                .take()
3340                .expect("outer evaluation resource request has active ledger");
3341            ledger.release_all_scratch();
3342            stats.ledger.update(ledger.snapshot());
3343            stats.outcome = match &result {
3344                Ok(_) => EvaluationRequestOutcome::Success,
3345                Err(error) if error.kind == ExcelErrorKind::Cancelled => {
3346                    EvaluationRequestOutcome::Cancelled
3347                }
3348                Err(_) => EvaluationRequestOutcome::Error,
3349            };
3350            if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
3351                stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
3352                    FormulaDirtyLeaseOutcome::RetainedOnCancellation
3353                } else {
3354                    FormulaDirtyLeaseOutcome::RetainedOnError
3355                };
3356            }
3357            stats.phases.total_ns = total_ns;
3358            let attributed = stats
3359                .phases
3360                .staged_prepare_ns
3361                .saturating_add(stats.phases.topology_ns)
3362                .saturating_add(stats.phases.materialization_ns);
3363            stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
3364            self.evaluation_resource_baseline.record_finished(&stats);
3365            self.last_evaluation_resource_request = Some(stats);
3366            crate::engine::trace::fz_event!(
3367                tracing::Level::INFO,
3368                "evaluate",
3369                "evaluate.summary",
3370                computed_vertices = self.trace_evaluation_counters.computed_vertices,
3371                cycles = self.trace_evaluation_counters.cycles,
3372                cancelled = matches!(
3373                    &result,
3374                    Err(error) if error.kind == ExcelErrorKind::Cancelled
3375                )
3376            );
3377        }
3378        result
3379    }
3380
3381    pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
3382        self.evaluation_resource_budgets = budgets.clone();
3383        self.config.evaluation_budgets = budgets.clone();
3384        self.graph.set_evaluation_budgets(budgets);
3385    }
3386
3387    #[cfg(test)]
3388    pub(crate) fn set_evaluation_budgets_for_test(
3389        &mut self,
3390        budgets: crate::engine::EvaluationBudgets,
3391    ) {
3392        self.set_evaluation_resource_budgets(budgets);
3393    }
3394
3395    fn preflight_evaluation_commit_window(
3396        &mut self,
3397        bounded_writes: usize,
3398    ) -> Result<crate::instant::FzInstant, ExcelError> {
3399        #[cfg(test)]
3400        if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
3401            return Err(crate::engine::ResourceLedgerError::Exhausted(
3402                formualizer_common::ResourceExhaustionDetail {
3403                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3404                    limit: 0,
3405                    observed: 1,
3406                    request_id: self
3407                        .active_evaluation_resource_request
3408                        .as_ref()
3409                        .map(|stats| stats.request_id),
3410                },
3411            )
3412            .into_excel_error());
3413        }
3414        let estimate = std::time::Duration::from_nanos(
3415            u64::try_from(bounded_writes)
3416                .unwrap_or(u64::MAX)
3417                .saturating_mul(100),
3418        );
3419        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3420            ledger
3421                .preflight_commit_window(estimate)
3422                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3423        }
3424        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3425            stats.evaluation_commit_preflight_count =
3426                stats.evaluation_commit_preflight_count.saturating_add(1);
3427            stats.evaluation_commit_estimated_ns = stats
3428                .evaluation_commit_estimated_ns
3429                .saturating_add(Self::duration_ns(estimate));
3430        }
3431        Ok(crate::instant::FzInstant::now())
3432    }
3433
3434    fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3435        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3436            stats.evaluation_commit_actual_ns = stats
3437                .evaluation_commit_actual_ns
3438                .saturating_add(Self::duration_ns(started.elapsed()));
3439        }
3440    }
3441
3442    /// Post-work cancellation boundary: call after evaluating a unit (or a
3443    /// parallel group) and before committing it. A function that observed
3444    /// the request's token returns `Err(Cancelled)`, which the evaluator
3445    /// turns into a `#CANCELLED` value; any other result computed across the
3446    /// signal is equally not a finished result. Neither may publish: the
3447    /// caller returns this error before committing, so the unit stays dirty,
3448    /// the request reports `Cancelled`, and a failed pass restores the
3449    /// vertices it already committed (`freshness_abort_pass`). The token is
3450    /// the discriminator: a `#CANCELLED` value with no live cancellation is
3451    /// ordinary data and commits. Deadlines are not checked here; finished
3452    /// work is not discarded on a deadline.
3453    fn live_cancellation_after_work(&self, message: &'static str) -> Result<(), ExcelError> {
3454        if self
3455            .active_cancel_flag
3456            .as_ref()
3457            .is_some_and(|cancel| cancel.is_cancelled())
3458        {
3459            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3460        }
3461        Ok(())
3462    }
3463
3464    fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3465        if self
3466            .active_cancel_flag
3467            .as_ref()
3468            .is_some_and(|cancel| cancel.is_cancelled())
3469        {
3470            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3471        }
3472        if self
3473            .active_evaluation_deadline
3474            .is_some_and(|deadline| Instant::now() >= deadline)
3475        {
3476            return Err(crate::engine::ResourceLedgerError::Exhausted(
3477                formualizer_common::ResourceExhaustionDetail {
3478                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3479                    limit: 0,
3480                    observed: 1,
3481                    request_id: self
3482                        .active_evaluation_resource_request
3483                        .as_ref()
3484                        .map(|request| request.request_id),
3485                },
3486            )
3487            .into_excel_error()
3488            .with_message(message));
3489        }
3490        Ok(())
3491    }
3492
3493    fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3494        let Some(ledger) = self.active_resource_ledger.as_mut() else {
3495            return Ok(());
3496        };
3497        ledger
3498            .charge_work(work_units)
3499            .and_then(|()| ledger.checkpoint_deadline())
3500            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3501    }
3502
3503    fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3504        if work_units == 0 {
3505            return self.resource_checkpoint(0);
3506        }
3507        while work_units > 0 {
3508            let chunk = work_units.min(256);
3509            self.resource_checkpoint(chunk)?;
3510            work_units -= chunk;
3511        }
3512        Ok(())
3513    }
3514
3515    fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3516        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3517            // Exact request topology and activation of the scratch cap are C1b. C1a records
3518            // scoped ownership but must not introduce a new skip or terminal path.
3519            ledger.observe_scratch(bytes);
3520        }
3521        Ok(())
3522    }
3523
3524    fn release_request_scratch(&mut self, bytes: u64) {
3525        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3526            let released = ledger.release_scratch(bytes);
3527            debug_assert!(
3528                released.is_ok(),
3529                "request scratch release exceeded the outstanding reservation"
3530            );
3531        }
3532    }
3533
3534    fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3535        self.active_resource_ledger
3536            .as_mut()
3537            .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3538            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3539    }
3540
3541    fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3542        self.active_resource_ledger
3543            .as_mut()
3544            .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3545            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3546    }
3547
3548    fn with_request_scratch<T>(
3549        &mut self,
3550        bytes: u64,
3551        work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3552    ) -> Result<T, ExcelError> {
3553        self.reserve_request_scratch(bytes)?;
3554        let result = work(self);
3555        self.release_request_scratch(bytes);
3556        result
3557    }
3558
3559    fn observe_staged_preparation(
3560        &mut self,
3561        selected: usize,
3562        retained: usize,
3563        elapsed: std::time::Duration,
3564    ) {
3565        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3566            stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3567            stats.staged_retained = retained as u64;
3568            stats.phases.staged_prepare_ns = stats
3569                .phases
3570                .staged_prepare_ns
3571                .saturating_add(Self::duration_ns(elapsed));
3572        }
3573    }
3574
3575    fn graph_admission_enabled(&self) -> bool {
3576        crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3577    }
3578
3579    fn preflight_graph_admission(
3580        &mut self,
3581        usage: crate::engine::resource_ledger::GraphAdmission,
3582    ) -> Result<(), ExcelError> {
3583        let request_id = self
3584            .active_evaluation_resource_request
3585            .as_ref()
3586            .map(|stats| stats.request_id);
3587        crate::engine::resource_ledger::preflight_graph_admission(
3588            &self.evaluation_resource_budgets,
3589            usage,
3590            request_id,
3591        )
3592        .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3593    }
3594
3595    fn prepared_legacy_admission(
3596        &mut self,
3597        plan: &PreparedLegacyGraphPlan,
3598        materialization_cells: u64,
3599    ) -> Result<(), ExcelError> {
3600        if !self.graph_admission_enabled() {
3601            return Ok(());
3602        }
3603        let stats = self.graph.baseline_stats();
3604        let added_edges = plan.planned_edge_count().ok_or_else(|| {
3605            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3606        })?;
3607        let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3608            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3609        })?;
3610        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3611            final_vertices: stats
3612                .graph_vertex_count
3613                .checked_add(plan.new_vertex_count())
3614                .ok_or_else(|| {
3615                    ExcelError::new(ExcelErrorKind::NImpl)
3616                        .with_message("graph vertex count overflow")
3617                })?,
3618            final_edges: stats
3619                .graph_edge_count
3620                .checked_sub(removed_edges)
3621                .and_then(|count| count.checked_add(added_edges))
3622                .ok_or_else(|| {
3623                    ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3624                })?,
3625            materialization_cells,
3626            added_vertices: plan.new_vertex_count(),
3627            added_edges,
3628        })
3629    }
3630
3631    fn observe_target_admission_failure(
3632        &mut self,
3633        reason: formualizer_common::ResourceExhaustionReason,
3634    ) {
3635        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3636            stats.target_admission_failure = Some(reason);
3637        }
3638    }
3639
3640    fn observe_target_preparation_report(
3641        &mut self,
3642        report: &crate::engine::PreparedTargetGraphReport,
3643    ) {
3644        let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3645            let index = match reason {
3646                crate::engine::OpaqueReason::DynamicReference => 0,
3647                crate::engine::OpaqueReason::RuntimeTextReference => 1,
3648                crate::engine::OpaqueReason::UnknownFunction => 2,
3649                crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3650                crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3651                crate::engine::OpaqueReason::UnresolvedName => 5,
3652                crate::engine::OpaqueReason::UnresolvedTable => 6,
3653                crate::engine::OpaqueReason::FormulaName => 7,
3654                crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3655                crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3656                crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3657            };
3658            1u64 << index
3659        };
3660        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3661            stats.staged_selected = report.selected_staged_cells as u64;
3662            stats.staged_retained = report.retained_staged_cells as u64;
3663            stats.target_requested = report.requested_targets as u64;
3664            stats.target_normalized_regions = report.normalized_regions as u64;
3665            stats.target_scope_level = match &report.widened_scope {
3666                crate::engine::PrepareScope::Exact => 0,
3667                crate::engine::PrepareScope::Sheets(_) => 1,
3668                crate::engine::PrepareScope::Workbook => 2,
3669            };
3670            stats.target_widening_reason_bits = report
3671                .widening_reasons
3672                .iter()
3673                .copied()
3674                .fold(0, |bits, reason| bits | reason_bit(reason));
3675            stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3676            stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3677            stats.target_commit_estimated_work = report.estimated_commit_work;
3678            stats.target_commit_actual_work = report.actual_commit_work;
3679            stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3680            stats.phases.staged_prepare_ns = stats
3681                .phases
3682                .staged_prepare_ns
3683                .saturating_add(Self::duration_ns(report.commit_window));
3684        }
3685    }
3686
3687    /// Begin a new evaluation request: reset per-recalc cycle telemetry and
3688    /// take the per-recalc volatile clock sample. Called at the start of
3689    /// every evaluation request that walks schedule units.
3690    fn begin_evaluation_request(&mut self) {
3691        self.freshness_begin_request();
3692        #[cfg(test)]
3693        {
3694            self.evaluation_request_begin_count_for_test = self
3695                .evaluation_request_begin_count_for_test
3696                .saturating_add(1);
3697        }
3698        self.last_cycle_telemetry = CycleTelemetry::default();
3699        self.graph.authority_sync();
3700        // Defensive: consumed at the end of the previous request; a request
3701        // that errored out mid-walk must not leak its members into this one.
3702        self.pending_iterative_redirty.clear();
3703        self.reconcile_retained_sccs_at_request_begin();
3704        // Spec §7.11: NOW()/TODAY() sample the clock ONCE per recalc; every
3705        // read within this request (including SCC iteration passes) observes
3706        // this sample.
3707        self.clock.refresh();
3708    }
3709
3710    /// End-of-recalc redirty: volatile vertices (as always) plus members of
3711    /// SCCs that iterated this recalc without reaching a retainable fixed
3712    /// point (`CyclePolicy::Iterate`), so circular cells re-evaluate on every
3713    /// recalc exactly like Excel's iterative calculation (spec §4
3714    /// persistence / §7.6 accumulator / §7.11 volatile redirty). Retained
3715    /// SCCs (`retained_scc_members`, #368) are left clean. Replaces the bare
3716    /// `graph.redirty_volatiles()` call at every evaluation-flow exit; must
3717    /// run AFTER the flow's `clear_dirty_flags`.
3718    fn redirty_for_next_recalc(&mut self) {
3719        if self.snapshot_evaluation_active {
3720            self.snapshot_volatile_redirty_pending = true;
3721        } else {
3722            self.graph.redirty_volatiles();
3723        }
3724        let pending = std::mem::take(&mut self.pending_iterative_redirty);
3725        let dirty_at_begin = std::mem::take(&mut self.retained_scc_dirty_at_begin);
3726        for (vertex, scc) in dirty_at_begin {
3727            if self.retained_scc_members.get(&vertex) == Some(&scc) {
3728                // No SCC task claimed this member during the request: the
3729                // cycle dissolved and it evaluated as an ordinary formula,
3730                // or the request never reached it. Either way it is no
3731                // longer a retained fixed point. Its persisted value is
3732                // obsolete only if it actually re-evaluated (clean now).
3733                self.retained_scc_members.remove(&vertex);
3734                if !self.graph.is_dirty(vertex) {
3735                    self.iterative_state_values.remove(&vertex);
3736                }
3737            }
3738        }
3739        if !self.iterative_state_values.is_empty() || !self.retained_scc_members.is_empty() {
3740            let graph = &self.graph;
3741            self.iterative_state_values
3742                .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3743            self.retained_scc_members
3744                .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3745        }
3746        // Refresh the §4-persistence snapshot for members that re-run each
3747        // recalc: these final values survive structural edits that clear the
3748        // computed overlay (the only value home in canonical mode) so the
3749        // next SCC task can re-seed from them (see `iterative_state_values`).
3750        for &vertex in &pending {
3751            if !self.graph.is_live_formula_vertex(vertex) {
3752                continue;
3753            }
3754            if let Some(cell) = self.graph.get_cell_ref(vertex) {
3755                let sheet_name = self.graph.sheet_name(cell.sheet_id);
3756                match self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1) {
3757                    Some(value) if !matches!(value, LiteralValue::Empty) => {
3758                        self.iterative_state_values.insert(vertex, value);
3759                    }
3760                    _ => {
3761                        self.iterative_state_values.remove(&vertex);
3762                    }
3763                }
3764            }
3765        }
3766        if !pending.is_empty() {
3767            self.graph.redirty_iterative_members(&pending);
3768        }
3769    }
3770
3771    /// Hash of every `EvalConfig` knob that can change the result of a
3772    /// retained SCC without any edit reaching the dependency graph. The
3773    /// function registry is tracked separately and precisely (see
3774    /// `retained_scc_function_epoch_seen`).
3775    fn retained_scc_config_fingerprint(&self) -> u64 {
3776        use std::hash::{Hash, Hasher};
3777        let mut hasher = rustc_hash::FxHasher::default();
3778        let config = &self.config;
3779        std::mem::discriminant(&config.cycle.detection).hash(&mut hasher);
3780        match config.cycle.policy {
3781            CyclePolicy::Error => 0u8.hash(&mut hasher),
3782            CyclePolicy::Iterate {
3783                max_iterations,
3784                max_change,
3785            } => {
3786                1u8.hash(&mut hasher);
3787                max_iterations.hash(&mut hasher);
3788                max_change.to_bits().hash(&mut hasher);
3789            }
3790        }
3791        config.date_system.hash(&mut hasher);
3792        config.workbook_seed.hash(&mut hasher);
3793        std::mem::discriminant(&config.volatile_level).hash(&mut hasher);
3794        // `DeterministicMode` carries a timestamp and a timezone spec; hash
3795        // its Debug rendering rather than growing its derive set for this.
3796        format!("{:?}", config.deterministic_mode).hash(&mut hasher);
3797        config.range_expansion_limit.hash(&mut hasher);
3798        config.max_open_ended_rows.hash(&mut hasher);
3799        config.max_open_ended_cols.hash(&mut hasher);
3800        hasher.finish()
3801    }
3802
3803    /// Request-begin bookkeeping for retained SCCs (#368): drop deleted
3804    /// vertices, invalidate everything when the config fingerprint moved
3805    /// (marking retained members dirty so their SCC tasks run in this
3806    /// request), and record how many retained SCCs are being reused, i.e.
3807    /// have no dirty member at request begin.
3808    fn reconcile_retained_sccs_at_request_begin(&mut self) {
3809        self.retained_scc_dirty_at_begin.clear();
3810        if self.retained_scc_members.is_empty() {
3811            return;
3812        }
3813        // Deleted vertices and members overwritten with a literal (the
3814        // vertex survives as a value cell) are no longer retained formulas.
3815        let graph = &self.graph;
3816        self.retained_scc_members
3817            .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3818        if self.retained_scc_config_fingerprint() != self.retained_scc_config_fingerprint {
3819            let members: Vec<VertexId> = self.retained_scc_members.keys().copied().collect();
3820            self.retained_scc_members.clear();
3821            self.graph.mark_dirty_many(&members);
3822            return;
3823        }
3824        let changes =
3825            crate::function_registry::semantic_changes_since(self.retained_scc_function_epoch_seen);
3826        let global_changed = changes.epoch != self.retained_scc_function_epoch_seen;
3827        let provider_revision = self.resolver.planning_semantic_revision();
3828        let provider_changed = provider_revision != self.retained_scc_provider_revision_seen;
3829        if global_changed || provider_changed {
3830            let changed: BTreeSet<(String, String)> = changes.keys.into_iter().collect();
3831            let affected: Vec<VertexId> = self
3832                .retained_scc_members
3833                .keys()
3834                .copied()
3835                .filter(|&vertex| {
3836                    let Some(ast) = self.graph.get_formula(vertex) else {
3837                        return true;
3838                    };
3839                    (global_changed
3840                        && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed)))
3841                        || (provider_changed && Self::ast_contains_function(&ast))
3842                })
3843                .collect();
3844            for vertex in &affected {
3845                self.retained_scc_members.remove(vertex);
3846            }
3847            if !affected.is_empty() {
3848                self.graph.mark_dirty_many(&affected);
3849            }
3850            self.retained_scc_function_epoch_seen = changes.epoch;
3851            self.retained_scc_provider_revision_seen = provider_revision;
3852        }
3853        let mut dirty_sccs: FxHashSet<u64> = FxHashSet::default();
3854        let mut all_sccs: FxHashSet<u64> = FxHashSet::default();
3855        for (&vertex, &scc) in &self.retained_scc_members {
3856            all_sccs.insert(scc);
3857            if self.graph.is_dirty(vertex) {
3858                dirty_sccs.insert(scc);
3859                self.retained_scc_dirty_at_begin.push((vertex, scc));
3860            }
3861        }
3862        let reused_members = self
3863            .retained_scc_members
3864            .values()
3865            .filter(|scc| !dirty_sccs.contains(scc))
3866            .count();
3867        let t = &mut self.last_cycle_telemetry;
3868        t.reused_sccs = all_sccs.len() - dirty_sccs.len();
3869        t.reused_scc_members = reused_members;
3870    }
3871
3872    pub fn virtual_dep_fallback_activations(&self) -> u64 {
3873        self.virtual_dep_fallback_activations
3874    }
3875
3876    #[cfg(test)]
3877    pub(crate) fn lookup_index_flights_built_for_test(&self) -> usize {
3878        self.lookup_index_cache
3879            .flights_built
3880            .load(std::sync::atomic::Ordering::Relaxed)
3881    }
3882
3883    pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
3884        self.lookup_index_cache.report()
3885    }
3886
3887    fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
3888        let start_row = view.start_row();
3889        let end_row = view.end_row();
3890        let start_col = view.start_col();
3891        let end_col = view.end_col();
3892        for row in start_row..=end_row {
3893            let Ok(row_u32) = u32::try_from(row) else {
3894                return true;
3895            };
3896            for col in start_col..=end_col {
3897                let Ok(col_u32) = u32::try_from(col) else {
3898                    return true;
3899                };
3900                let cell_ref = self
3901                    .graph
3902                    .make_cell_ref_internal(sheet_id, row_u32, col_u32);
3903                if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
3904                    && self.graph.is_volatile(vertex_id)
3905                {
3906                    return true;
3907                }
3908            }
3909        }
3910        false
3911    }
3912
3913    fn build_lookup_index_impl(
3914        &self,
3915        view: &RangeView<'_>,
3916        axis: LookupAxis,
3917    ) -> Option<Arc<LookupIndex>> {
3918        let (rows, cols) = view.dims();
3919        if rows == 0 || cols == 0 {
3920            self.lookup_index_cache.note_skipped_tiny();
3921            return None;
3922        }
3923        let len = match axis {
3924            LookupAxis::ColumnInView(col) => {
3925                if col >= cols {
3926                    self.lookup_index_cache.note_skipped_tiny();
3927                    return None;
3928                }
3929                rows
3930            }
3931            LookupAxis::RowInView(row) => {
3932                if row >= rows {
3933                    self.lookup_index_cache.note_skipped_tiny();
3934                    return None;
3935                }
3936                cols
3937            }
3938        };
3939        if len < 64 {
3940            self.lookup_index_cache.note_skipped_tiny();
3941            return None;
3942        }
3943
3944        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
3945        let key = LookupIndexKey {
3946            sheet_id,
3947            start_row: u32::try_from(view.start_row()).ok()?,
3948            start_col: u32::try_from(view.start_col()).ok()?,
3949            end_row: u32::try_from(view.end_row()).ok()?,
3950            end_col: u32::try_from(view.end_col()).ok()?,
3951            axis,
3952            snapshot_id: self.data_snapshot_id(),
3953        };
3954        if let Some(index) = self.lookup_index_cache.get(&key) {
3955            return Some(index);
3956        }
3957        if self
3958            .lookup_index_cache
3959            .would_exceed_cap(estimate_bytes(len, 0))
3960        {
3961            self.lookup_index_cache.note_skipped_cap();
3962            return None;
3963        }
3964        if !self.lookup_index_cache.should_build(key) {
3965            return None;
3966        }
3967        // Parallel members of a lookup family miss together: one builds.
3968        self.lookup_index_cache.single_flight(key, || {
3969            if let Some(index) = self.lookup_index_cache.recheck(&key) {
3970                return Some(index);
3971            }
3972            if self.lookup_index_cache.is_known_volatile(&key) {
3973                self.lookup_index_cache.note_skipped_volatile();
3974                return None;
3975            }
3976            if self.lookup_view_contains_volatile(view, sheet_id) {
3977                self.lookup_index_cache.note_volatile_key(key);
3978                self.lookup_index_cache.note_skipped_volatile();
3979                return None;
3980            }
3981            #[cfg(test)]
3982            self.lookup_index_cache
3983                .flights_built
3984                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3985            match LookupIndex::build(view, axis, self.config.date_system).ok()? {
3986                BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
3987                BuildOutcome::ErrorInLookupAxis => {
3988                    self.lookup_index_cache.note_skipped_error();
3989                    None
3990                }
3991                BuildOutcome::Degenerate => {
3992                    self.lookup_index_cache.note_skipped_tiny();
3993                    None
3994                }
3995            }
3996        })
3997    }
3998
3999    fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
4000        if !self.config.enable_virtual_dep_telemetry {
4001            self.last_virtual_dep_telemetry = VirtualDepTelemetry {
4002                fallback_mode_activations: self.virtual_dep_fallback_activations,
4003                ..VirtualDepTelemetry::default()
4004            };
4005        }
4006    }
4007
4008    fn source_cache_session(&self) -> SourceCacheSession {
4009        self.clear_source_cache();
4010        SourceCacheSession {
4011            cache: self.source_cache.clone(),
4012        }
4013    }
4014
4015    fn resolve_source_scalar_cached(
4016        &self,
4017        name: &str,
4018        version: Option<u64>,
4019    ) -> Result<LiteralValue, ExcelError> {
4020        let key = (name.to_string(), version);
4021        if let Ok(mut g) = self.source_cache.write() {
4022            if let Some(v) = g.scalars.get(&key) {
4023                return Ok(v.clone());
4024            }
4025
4026            let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
4027                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4028                    ExcelError::new(ExcelErrorKind::Ref)
4029                        .with_message(format!("Unresolved source scalar: {name}"))
4030                } else {
4031                    err
4032                }
4033            })?;
4034            g.scalars.insert(key, v.clone());
4035            Ok(v)
4036        } else {
4037            self.resolver.resolve_source_scalar(name).map_err(|err| {
4038                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4039                    ExcelError::new(ExcelErrorKind::Ref)
4040                        .with_message(format!("Unresolved source scalar: {name}"))
4041                } else {
4042                    err
4043                }
4044            })
4045        }
4046    }
4047
4048    fn resolve_source_table_cached(
4049        &self,
4050        name: &str,
4051        version: Option<u64>,
4052    ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
4053        let key = (name.to_string(), version);
4054        if let Ok(mut g) = self.source_cache.write() {
4055            if let Some(t) = g.tables.get(&key) {
4056                return Ok(t.clone());
4057            }
4058
4059            let t = self.resolver.resolve_source_table(name).map_err(|err| {
4060                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4061                    ExcelError::new(ExcelErrorKind::Ref)
4062                        .with_message(format!("Unresolved source table: {name}"))
4063                } else {
4064                    err
4065                }
4066            })?;
4067            let t: Arc<dyn crate::traits::Table> = Arc::from(t);
4068            g.tables.insert(key, t.clone());
4069            Ok(t)
4070        } else {
4071            self.resolver
4072                .resolve_source_table(name)
4073                .map_err(|err| {
4074                    if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4075                        ExcelError::new(ExcelErrorKind::Ref)
4076                            .with_message(format!("Unresolved source table: {name}"))
4077                    } else {
4078                        err
4079                    }
4080                })
4081                .map(Arc::from)
4082        }
4083    }
4084
4085    fn source_table_to_range_view(
4086        &self,
4087        table: &dyn crate::traits::Table,
4088        spec: &Option<formualizer_parse::parser::TableSpecifier>,
4089    ) -> Result<RangeView<'static>, ExcelError> {
4090        use formualizer_parse::parser::{SpecialItem, TableSpecifier};
4091
4092        let owned = match spec {
4093            Some(TableSpecifier::Column(c)) => {
4094                let c = c.trim();
4095                if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
4096                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4097                        "Complex structured references not yet supported".to_string(),
4098                    ));
4099                }
4100                table.get_column(c)?.materialise().into_owned()
4101            }
4102            Some(TableSpecifier::ColumnRange(start, end)) => {
4103                let cols = table.columns();
4104                let start = start.trim();
4105                let end = end.trim();
4106                let start_key = start.to_lowercase();
4107                let end_key = end.to_lowercase();
4108                let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
4109                let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
4110                if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
4111                    if si > ei {
4112                        std::mem::swap(&mut si, &mut ei);
4113                    }
4114                    let h = table.data_height();
4115                    let w = ei - si + 1;
4116                    let mut rows = vec![vec![LiteralValue::Empty; w]; h];
4117                    for (offset, ci) in (si..=ei).enumerate() {
4118                        let cname = &cols[ci];
4119                        let col_range = table.get_column(cname)?;
4120                        let (rh, _) = col_range.dimensions();
4121                        for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
4122                            row[offset] = col_range.get(r, 0)?;
4123                        }
4124                    }
4125                    rows
4126                } else {
4127                    return Err(ExcelError::new(ExcelErrorKind::Ref)
4128                        .with_message("Column range refers to unknown column(s)".to_string()));
4129                }
4130            }
4131            Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
4132            | Some(TableSpecifier::Headers) => table
4133                .headers_row()
4134                .map(|r| r.materialise().into_owned())
4135                .unwrap_or_default(),
4136            Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
4137            | Some(TableSpecifier::Totals) => table
4138                .totals_row()
4139                .map(|r| r.materialise().into_owned())
4140                .unwrap_or_default(),
4141            Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
4142                table
4143                    .data_body()
4144                    .map(|r| r.materialise().into_owned())
4145                    .unwrap_or_default()
4146            }
4147            Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
4148                let mut out: Vec<Vec<LiteralValue>> = Vec::new();
4149                if let Some(h) = table.headers_row() {
4150                    out.extend(h.iter_rows());
4151                }
4152                if let Some(body) = table.data_body() {
4153                    out.extend(body.iter_rows());
4154                }
4155                if let Some(tr) = table.totals_row() {
4156                    out.extend(tr.iter_rows());
4157                }
4158                out
4159            }
4160            Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
4161                return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4162                    "@ (This Row) requires table-aware context; not yet supported".to_string(),
4163                ));
4164            }
4165            Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
4166                return Err(ExcelError::new(ExcelErrorKind::NImpl)
4167                    .with_message("Complex structured references not yet supported".to_string()));
4168            }
4169            None => {
4170                return Err(ExcelError::new(ExcelErrorKind::NImpl)
4171                    .with_message("Table reference without specifier is unsupported".to_string()));
4172            }
4173        };
4174
4175        Ok(RangeView::from_owned_rows(owned, self.config.date_system))
4176    }
4177
4178    pub fn default_sheet_id(&self) -> SheetId {
4179        self.graph.default_sheet_id()
4180    }
4181
4182    pub fn default_sheet_name(&self) -> &str {
4183        self.graph.default_sheet_name()
4184    }
4185
4186    /// Update the workbook seed for deterministic RNGs in functions.
4187    pub fn set_workbook_seed(&mut self, seed: u64) {
4188        self.config.workbook_seed = seed;
4189    }
4190
4191    /// Set the volatile level policy (Always/OnRecalc/OnOpen)
4192    pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
4193        self.config.volatile_level = level;
4194    }
4195
4196    /// Set public temporal materialisation to native values or raw serials.
4197    pub fn set_temporal_egress(&mut self, policy: crate::engine::TemporalEgress) {
4198        self.config.temporal_egress = policy;
4199    }
4200
4201    pub fn temporal_egress(&self) -> crate::engine::TemporalEgress {
4202        self.config.temporal_egress
4203    }
4204
4205    /// Enable/disable deterministic evaluation mode (fixed clock + timezone).
4206    pub fn set_deterministic_mode(
4207        &mut self,
4208        mode: crate::engine::DeterministicMode,
4209    ) -> Result<(), ExcelError> {
4210        let clock = mode.build_clock()?;
4211        self.config.deterministic_mode = mode;
4212        self.clock = crate::timezone::SnapshotClock::new(clock);
4213        Ok(())
4214    }
4215
4216    /// Inject a custom [`ClockProvider`](crate::timezone::ClockProvider) for
4217    /// volatile date/time builtins (`NOW()`, `TODAY()`).
4218    ///
4219    /// The provider is the clock *source*; per spec §7.11 the engine samples
4220    /// it once at the start of every evaluation request and all reads within
4221    /// that recalc (including SCC iteration passes) observe the frozen
4222    /// sample.
4223    pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
4224        self.clock = crate::timezone::SnapshotClock::new(clock);
4225    }
4226
4227    fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
4228        self.config.deterministic_mode.validate()
4229    }
4230
4231    pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
4232        self.graph.sheet_id(name)
4233    }
4234
4235    pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
4236        self.add_sheet(name)
4237            .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
4238    }
4239
4240    pub fn sheet_name(&self, id: SheetId) -> &str {
4241        self.graph.sheet_name(id)
4242    }
4243
4244    pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
4245        let id = self.graph.add_sheet(name)?;
4246        self.ensure_arrow_sheet(name);
4247        self.mark_topology_edited();
4248        Ok(id)
4249    }
4250
4251    pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
4252        let source_id = self.graph.sheet_id(source).ok_or_else(|| {
4253            ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
4254        })?;
4255        if new_name.is_empty() || new_name.len() > 255 {
4256            return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
4257        }
4258        if self.graph.sheet_id(new_name).is_some() {
4259            return Err(ExcelError::new(ExcelErrorKind::Value)
4260                .with_message(format!("Sheet '{new_name}' already exists")));
4261        }
4262        let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
4263
4264        if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
4265            let mut copied_sheet = source_sheet;
4266            copied_sheet.name = Arc::<str>::from(new_name);
4267            self.arrow_sheets.sheets.push(copied_sheet);
4268        } else {
4269            self.ensure_arrow_sheet(new_name);
4270        }
4271
4272        let duplicated_formulas = self
4273            .graph
4274            .formula_vertices()
4275            .into_iter()
4276            .filter(|vertex| {
4277                self.graph
4278                    .get_cell_ref(*vertex)
4279                    .is_some_and(|cell| cell.sheet_id == new_id)
4280            })
4281            .collect::<Vec<_>>();
4282        self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
4283        self.mark_topology_edited();
4284        Ok(new_id)
4285    }
4286
4287    fn ensure_arrow_sheet(&mut self, name: &str) {
4288        if self.arrow_sheets.sheet(name).is_some() {
4289            return;
4290        }
4291        self.arrow_sheets
4292            .sheets
4293            .push(crate::arrow_store::ArrowSheet {
4294                name: std::sync::Arc::<str>::from(name),
4295                date_system: self.config.date_system,
4296                columns: Vec::new(),
4297                nrows: 0,
4298                chunk_starts: Vec::new(),
4299                chunk_rows: 32 * 1024,
4300            });
4301    }
4302
4303    pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
4304        let name = self.graph.sheet_name(sheet_id).to_string();
4305        self.purge_derived_formats_for_sheet(sheet_id);
4306        self.graph.remove_sheet(sheet_id)?;
4307        self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
4308        // Sheet removal can change cross-sheet refs, names, and default-sheet
4309        // resolution. Until those domains have a complete exact dependency
4310        // proof, retain the documented graph-owned global invalidation.
4311        self.clear_all_computed_overlays();
4312        self.mark_all_formula_vertices_dirty();
4313        self.clear_staged_formulas_for_sheet(&name);
4314        if self.row_visibility.remove(&sheet_id).is_some() {
4315            self.invalidate_row_visibility_mask_cache();
4316        }
4317        self.record_structural_change(StructuralScope::RemovedSheet(sheet_id));
4318        self.mark_topology_edited();
4319        Ok(())
4320    }
4321
4322    /// Helper to synchronize the Arrow-backed storage layer.
4323    fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
4324        if let Some(asheet) = self
4325            .arrow_sheets
4326            .sheets
4327            .iter_mut()
4328            .find(|s| s.name.as_ref() == target_name)
4329        {
4330            asheet.name = std::sync::Arc::<str>::from(new_name);
4331            return true;
4332        }
4333        false
4334    }
4335
4336    pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4337        let old_name = self.graph.sheet_name(sheet_id).to_string();
4338
4339        // Speculative Storage Update
4340        // Update name in storage FIRST so the Evaluator can find it during Graph rescue.
4341        self.rename_sheet_in_arrow_store(&old_name, new_name);
4342
4343        // Graph Update (Metadata + Rescue Logic)
4344        match self.graph.rename_sheet(sheet_id, new_name) {
4345            Ok(_) => {
4346                self.rename_staged_formula_sheet(&old_name, new_name);
4347                // Success! Invalidate cache for the moved sheet
4348                let sheet_vertices: Vec<VertexId> = self
4349                    .graph
4350                    .grid_vertices_in_sheet(sheet_id)
4351                    .map(|(id, _)| id)
4352                    .collect();
4353                for v_id in sheet_vertices {
4354                    self.graph.mark_vertex_dirty(v_id);
4355                }
4356                // Sheet rename preserves SheetId and therefore formula dependencies.
4357                self.mark_topology_edited();
4358                Ok(())
4359            }
4360            Err(e) => {
4361                // ROLLBACK: Revert storage if graph rejected the name
4362                self.rename_sheet_in_arrow_store(new_name, &old_name);
4363                Err(e)
4364            }
4365        }
4366    }
4367
4368    pub fn named_ranges_iter(
4369        &self,
4370    ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4371        self.graph.named_ranges_iter()
4372    }
4373
4374    pub fn sheet_named_ranges_iter(
4375        &self,
4376    ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4377        self.graph.sheet_named_ranges_iter()
4378    }
4379
4380    pub fn resolve_name_entry(
4381        &self,
4382        name: &str,
4383        current_sheet: SheetId,
4384    ) -> Option<&crate::engine::named_range::NamedRange> {
4385        self.graph.resolve_name_entry(name, current_sheet)
4386    }
4387
4388    /// The [`NameScope`] an optional scope-sheet argument denotes.
4389    ///
4390    /// `None` means **workbook scope**, not "the default sheet": a caller that
4391    /// supplies no sheet context is asking about workbook-scoped names only.
4392    /// An unknown sheet name is a malformed query and errors rather than
4393    /// silently degrading to another sheet's scope (issue #110).
4394    ///
4395    /// This is the one owned derivation from `Option<&str>` to a name scope;
4396    /// every scope-taking entry point routes through it.
4397    pub(crate) fn name_query_scope(
4398        &self,
4399        scope_sheet: Option<&str>,
4400    ) -> Result<NameScope, ExcelError> {
4401        match scope_sheet {
4402            None => Ok(NameScope::Workbook),
4403            Some(sheet) => self
4404                .graph
4405                .sheet_id(sheet)
4406                .map(NameScope::Sheet)
4407                .ok_or_else(|| {
4408                    ExcelError::new(ExcelErrorKind::Ref)
4409                        .with_message(format!("name scope sheet not found: {sheet}"))
4410                }),
4411        }
4412    }
4413
4414    /// Whether `name` resolves in the scope denoted by `scope_sheet`.
4415    ///
4416    /// `scope_sheet == None` asks about workbook scope only; a name scoped to a
4417    /// single sheet (including the default sheet) does not answer it. An unknown
4418    /// sheet name resolves nothing.
4419    /// Resolve a [`SharedSheetLocator`](crate::reference::SharedSheetLocator)
4420    /// against an explicit context sheet.
4421    ///
4422    /// Thin forwarder to
4423    /// [`SheetRegistry::resolve_locator`](crate::engine::sheet_registry::SheetRegistry::resolve_locator),
4424    /// the single owned derivation. `Current` resolves to `context_sheet`, never
4425    /// to the workbook's default sheet.
4426    fn resolve_sheet_locator(
4427        &self,
4428        locator: &crate::reference::SharedSheetLocator<'_>,
4429        context_sheet: SheetId,
4430    ) -> Result<SheetId, ExcelError> {
4431        self.graph
4432            .sheet_reg()
4433            .resolve_locator(locator, context_sheet)
4434    }
4435
4436    pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4437        let Ok(scope) = self.name_query_scope(scope_sheet) else {
4438            return false;
4439        };
4440        self.graph
4441            .resolve_name_entry_in_scope(name, scope)
4442            .is_some()
4443    }
4444
4445    /// The current value of `name` in the scope denoted by `scope_sheet`.
4446    ///
4447    /// Scoping follows [`Self::has_name`]: `None` is workbook scope only.
4448    pub fn resolved_name_value(
4449        &self,
4450        name: &str,
4451        scope_sheet: Option<&str>,
4452    ) -> Option<LiteralValue> {
4453        let scope = self.name_query_scope(scope_sheet).ok()?;
4454        let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4455        self.graph.get_value(entry.vertex)
4456    }
4457
4458    pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4459        let entry = self.graph.resolve_table_entry(name)?;
4460        Some(TableMetadata {
4461            name: entry.name.clone(),
4462            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4463            start_row: entry.range.start.coord.row() + 1,
4464            start_col: entry.range.start.coord.col() + 1,
4465            end_row: entry.range.end.coord.row() + 1,
4466            end_col: entry.range.end.coord.col() + 1,
4467            header_row: entry.header_row,
4468            headers: entry.headers.clone(),
4469            totals_row: entry.totals_row,
4470        })
4471    }
4472
4473    /// Metadata for every defined table, ordered by name.
4474    pub fn tables(&self) -> Vec<TableMetadata> {
4475        self.graph
4476            .table_names()
4477            .into_iter()
4478            .filter_map(|name| self.table_metadata(&name))
4479            .collect()
4480    }
4481
4482    pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4483        let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4484
4485        for (name, named) in self.graph.named_ranges_iter() {
4486            out.push(crate::engine::named_range::NamedRangeSnapshot {
4487                name: name.clone(),
4488                scope: NameScope::Workbook,
4489                definition: named.definition.clone(),
4490            });
4491        }
4492
4493        for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4494            out.push(crate::engine::named_range::NamedRangeSnapshot {
4495                name: name.clone(),
4496                scope: NameScope::Sheet(*sheet_id),
4497                definition: named.definition.clone(),
4498            });
4499        }
4500
4501        out.sort_by(|a, b| {
4502            let a_scope = match a.scope {
4503                NameScope::Workbook => (0u8, 0u32),
4504                NameScope::Sheet(id) => (1u8, u32::from(id)),
4505            };
4506            let b_scope = match b.scope {
4507                NameScope::Workbook => (0u8, 0u32),
4508                NameScope::Sheet(id) => (1u8, u32::from(id)),
4509            };
4510            a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4511        });
4512
4513        out
4514    }
4515
4516    pub fn named_ranges_snapshot_for_sheet(
4517        &self,
4518        sheet_id: SheetId,
4519    ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4520        self.named_ranges_snapshot()
4521            .into_iter()
4522            .filter(|entry| match entry.scope {
4523                NameScope::Workbook => true,
4524                NameScope::Sheet(id) => id == sheet_id,
4525            })
4526            .collect()
4527    }
4528
4529    pub fn define_name(
4530        &mut self,
4531        name: &str,
4532        definition: NamedDefinition,
4533        scope: NameScope,
4534    ) -> Result<(), ExcelError> {
4535        self.graph.validate_define_name(name, scope)?;
4536        self.graph.define_name(name, definition, scope)?;
4537        self.record_structural_change(StructuralScope::AllSheets);
4538
4539        self.mark_topology_edited();
4540
4541        Ok(())
4542    }
4543
4544    pub fn update_name(
4545        &mut self,
4546        name: &str,
4547        definition: NamedDefinition,
4548        scope: NameScope,
4549    ) -> Result<(), ExcelError> {
4550        self.graph.validate_existing_name(name, scope)?;
4551        self.graph.update_name(name, definition, scope)?;
4552        self.record_structural_change(StructuralScope::AllSheets);
4553
4554        self.mark_topology_edited();
4555
4556        Ok(())
4557    }
4558
4559    pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4560        self.graph.validate_existing_name(name, scope)?;
4561        self.graph.delete_name(name, scope)?;
4562        self.record_structural_change(StructuralScope::AllSheets);
4563
4564        self.mark_topology_edited();
4565
4566        Ok(())
4567    }
4568
4569    pub fn define_table(
4570        &mut self,
4571        name: &str,
4572        range: crate::reference::RangeRef,
4573        header_row: bool,
4574        headers: Vec<String>,
4575        totals_row: bool,
4576    ) -> Result<(), ExcelError> {
4577        self.graph
4578            .define_table(name, range, header_row, headers, totals_row)?;
4579        self.record_structural_change(StructuralScope::AllSheets);
4580        self.mark_topology_edited();
4581        Ok(())
4582    }
4583
4584    pub fn define_source_scalar(
4585        &mut self,
4586        name: &str,
4587        version: Option<u64>,
4588    ) -> Result<(), ExcelError> {
4589        self.graph.define_source_scalar(name, version)?;
4590        self.record_structural_change(StructuralScope::OpaqueGlobal);
4591        self.mark_topology_edited();
4592        Ok(())
4593    }
4594
4595    pub fn define_source_table(
4596        &mut self,
4597        name: &str,
4598        version: Option<u64>,
4599    ) -> Result<(), ExcelError> {
4600        self.graph.define_source_table(name, version)?;
4601        self.record_structural_change(StructuralScope::OpaqueGlobal);
4602        self.mark_topology_edited();
4603        Ok(())
4604    }
4605
4606    pub fn set_source_scalar_version(
4607        &mut self,
4608        name: &str,
4609        version: Option<u64>,
4610    ) -> Result<(), ExcelError> {
4611        self.graph.set_source_scalar_version(name, version)?;
4612        Ok(())
4613    }
4614
4615    pub fn set_source_table_version(
4616        &mut self,
4617        name: &str,
4618        version: Option<u64>,
4619    ) -> Result<(), ExcelError> {
4620        self.graph.set_source_table_version(name, version)?;
4621        Ok(())
4622    }
4623
4624    pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4625        self.graph.invalidate_source(name)?;
4626        Ok(())
4627    }
4628
4629    pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4630        self.graph.get_value(vertex)
4631    }
4632
4633    pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4634        self.graph.get_cell_value(sheet, row, col)
4635    }
4636
4637    pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4638        self.graph.get_vertex_for_cell(cell)
4639    }
4640
4641    pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4642        self.graph.get_evaluation_vertices()
4643    }
4644
4645    /// Return read-only baseline counters for dispatch benchmarking.
4646    pub fn baseline_stats(&self) -> EngineBaselineStats {
4647        let graph = self.graph.baseline_stats();
4648        EngineBaselineStats {
4649            graph_vertex_count: graph.graph_vertex_count,
4650            graph_formula_vertex_count: graph.graph_formula_vertex_count,
4651            graph_edge_count: graph.graph_edge_count,
4652            dirty_vertex_count: graph.dirty_vertex_count,
4653            evaluation_vertex_count: graph.evaluation_vertex_count,
4654            formula_ast_root_count: graph.formula_ast_root_count,
4655            formula_ast_node_count: graph.formula_ast_node_count,
4656            staged_formula_count: self.staged_formula_count(),
4657            formula_plane_active_span_count: 0,
4658            formula_plane_producer_result_entries: 0,
4659            formula_plane_consumer_read_entries: 0,
4660            formula_plane_mixed_topology_cache_builds: 0,
4661            formula_plane_mixed_topology_cache_hits: 0,
4662            formula_plane_mixed_topology_cache_overflows: 0,
4663            formula_plane_dirty_pending_events: 0,
4664            formula_plane_dirty_region_events_recorded: 0,
4665            formula_plane_dirty_span_region_events_recorded: 0,
4666            formula_plane_dirty_whole_span_seeds_recorded: 0,
4667            formula_plane_dirty_global_invalidations: 0,
4668            formula_plane_structural_span_candidates: 0,
4669            formula_plane_cycle_member_span_demotions: 0,
4670            formula_plane_array_result_span_demotions: 0,
4671            retained_scc_members: self.retained_scc_members.len(),
4672        }
4673    }
4674
4675    /// Mutation revision captured by read-only engine reports.
4676    pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4677        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4678    }
4679
4680    #[cfg(test)]
4681    pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4682        self.used_axis_bounds_cache
4683            .read()
4684            .ok()
4685            .and_then(|guard| {
4686                guard.as_ref().map(|cache| {
4687                    (
4688                        cache.row_hits.load(Ordering::Relaxed),
4689                        cache.row_misses.load(Ordering::Relaxed),
4690                        cache.col_hits.load(Ordering::Relaxed),
4691                        cache.col_misses.load(Ordering::Relaxed),
4692                    )
4693                })
4694            })
4695            .unwrap_or((0, 0, 0, 0))
4696    }
4697
4698    pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4699        self.graph.set_first_load_assume_new(enabled);
4700    }
4701
4702    pub fn first_load_assume_new(&self) -> bool {
4703        self.graph.first_load_assume_new()
4704    }
4705
4706    pub fn reset_ensure_touched(&mut self) {
4707        self.graph.reset_ensure_touched();
4708    }
4709
4710    pub fn finalize_sheet_index(&mut self, sheet: &str) {
4711        self.graph.finalize_sheet_index(sheet);
4712    }
4713
4714    /// Execute a named Engine action.
4715    ///
4716    /// Ticket 614 introduces this as the stable Engine-level transaction surface.
4717    /// For now actions are commit-only: they do not create changelog boundaries and they do not
4718    /// provide rollback/atomicity.
4719    ///
4720    /// Nested actions are deterministically handled by *disallowing* nesting: calling
4721    /// `Engine::action` while another action is active returns `EditorError::TransactionFailed`.
4722    pub fn action<T>(
4723        &mut self,
4724        name: impl AsRef<str>,
4725        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4726    ) -> Result<T, crate::engine::EditorError> {
4727        if self.action_depth != 0 {
4728            return Err(crate::engine::EditorError::TransactionFailed {
4729                reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4730                    .to_string(),
4731            });
4732        }
4733
4734        self.action_depth = 1;
4735        let engine_ptr: *mut Engine<R> = self;
4736        let _guard = ActionDepthGuard {
4737            engine: engine_ptr,
4738            _marker: std::marker::PhantomData,
4739        };
4740
4741        let mut tx = EngineAction {
4742            engine: self,
4743            name: name.as_ref().to_string(),
4744            capture: None,
4745            arrow_undo: None,
4746            atomic_policy: false,
4747        };
4748        f(&mut tx)
4749    }
4750
4751    /// Execute a named Engine action with atomic commit/rollback semantics.
4752    ///
4753    /// This variant does not require a `ChangeLog` and uses an internal journal for rollback.
4754    pub fn action_atomic<T>(
4755        &mut self,
4756        name: impl Into<String>,
4757        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4758    ) -> Result<T, crate::engine::EditorError> {
4759        let (v, _j) = self.action_atomic_journal(name, f)?;
4760        Ok(v)
4761    }
4762
4763    /// Like `action_atomic`, but returns the committed journal entry for undo/redo storage.
4764    pub fn action_atomic_journal<T>(
4765        &mut self,
4766        name: impl Into<String>,
4767        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4768    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4769        if self.action_depth != 0 {
4770            return Err(crate::engine::EditorError::TransactionFailed {
4771                reason: "Nested Engine::action calls are not supported (deterministic rule)"
4772                    .to_string(),
4773            });
4774        }
4775
4776        self.action_depth = 1;
4777        let engine_ptr: *mut Engine<R> = self;
4778        let _guard = ActionDepthGuard {
4779            engine: engine_ptr,
4780            _marker: std::marker::PhantomData,
4781        };
4782
4783        let name_str = name.into();
4784        let mut capture = MutationCapture::new(Default::default());
4785        let start_len = capture.len();
4786        self.action_atomic_impl(&mut capture, start_len, true, name_str, f)
4787    }
4788
4789    fn action_atomic_impl<T>(
4790        &mut self,
4791        capture: &mut MutationCapture,
4792        start_len: usize,
4793        expand_runs: bool,
4794        name: String,
4795        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4796    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4797        let invalidation_baseline = self.invalidation_baseline();
4798        let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4799        let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4800
4801        let capture_ptr: *mut MutationCapture = capture;
4802        let mut tx = EngineAction {
4803            engine: self,
4804            name: name.clone(),
4805            capture: Some(capture_ptr),
4806            arrow_undo: Some(arrow_ptr),
4807            atomic_policy: true,
4808        };
4809
4810        let res = f(&mut tx);
4811
4812        // Capture graph structural delta for this action. The journal is a
4813        // public value: run records (Program 2) are expanded into it, except
4814        // for a caller that only publishes the capture to its change log
4815        // (`expand_runs` false), where the journal (plain events) drives
4816        // invalidation and records count as topology changes; a rollback
4817        // still replays the expanded events.
4818        let capture_ref = unsafe { &*capture_ptr };
4819        let has_runs = capture_ref.lazy_len() > 0;
4820        let graph_events: Vec<crate::engine::ChangeEvent> = if expand_runs || res.is_err() {
4821            capture_ref.expanded_events_from(start_len, 0)
4822        } else {
4823            capture_ref.events()[start_len..].to_vec()
4824        };
4825        let graph_batch = crate::engine::GraphUndoBatch {
4826            events: graph_events,
4827        };
4828        let affected_cells = arrow_undo.ops.len();
4829        let journal = crate::engine::ActionJournal {
4830            name,
4831            graph: graph_batch,
4832            arrow: arrow_undo,
4833            affected_cells,
4834        };
4835
4836        match res {
4837            Ok(v) => {
4838                if !journal.graph.is_empty() || !journal.arrow.is_empty() || has_runs {
4839                    for event in &journal.graph.events {
4840                        self.record_change_for_event(event);
4841                    }
4842                    let mut impact = Self::classify_change_events(
4843                        &journal.graph.events,
4844                        LoggedEditDirection::Original,
4845                    )
4846                    .max(Self::classify_arrow_undo(&journal.arrow));
4847                    if has_runs {
4848                        impact = impact.max(LoggedEditImpact::Topology);
4849                    }
4850                    self.apply_logged_edit_impact(impact, invalidation_baseline);
4851                }
4852                Ok((v, journal))
4853            }
4854            Err(e) => {
4855                if let Err(rb) = self.rollback_from_action_journal(&journal, invalidation_baseline)
4856                {
4857                    return Err(crate::engine::EditorError::TransactionFailed {
4858                        reason: format!(
4859                            "Engine::action_atomic rollback failed after error '{e}': {rb}"
4860                        ),
4861                    });
4862                }
4863                if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4864                    for event in &journal.graph.events {
4865                        self.record_change_for_event(event);
4866                    }
4867                }
4868                Err(e)
4869            }
4870        }
4871    }
4872
4873    /// Execute a named Engine action, logging graph changes into the provided ChangeLog.
4874    ///
4875    /// Ticket 615: this variant provides atomicity. If the action returns an error, it rolls back:
4876    /// - Dependency graph structural edits (via inverse ChangeEvents)
4877    /// - Arrow-truth overlay writes mirrored from ChangeEvents
4878    /// - ChangeLog entries (published only after a successful commit)
4879    pub fn action_with_logger<T>(
4880        &mut self,
4881        log: &mut crate::engine::ChangeLog,
4882        name: impl AsRef<str>,
4883        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4884    ) -> Result<T, crate::engine::EditorError> {
4885        if self.action_depth != 0 {
4886            return Err(crate::engine::EditorError::TransactionFailed {
4887                reason: "Nested Engine::action calls are not supported (deterministic rule)"
4888                    .to_string(),
4889            });
4890        }
4891
4892        self.action_depth = 1;
4893        let engine_ptr: *mut Engine<R> = self;
4894        let _guard = ActionDepthGuard {
4895            engine: engine_ptr,
4896            _marker: std::marker::PhantomData,
4897        };
4898
4899        let name_str = name.as_ref().to_string();
4900        let mut capture = MutationCapture::new(log.current_meta());
4901        let start_len = capture.len();
4902        capture.begin_compound(name_str.clone());
4903
4904        // Mutation correctness uses the complete private capture. The provided ChangeLog remains
4905        // an observability sink and is not touched until the action outcome is known.
4906        let res = self.action_atomic_impl(&mut capture, start_len, false, name_str, f);
4907        capture.close_compounds();
4908
4909        match res {
4910            Ok((v, _journal)) => {
4911                log.publish_capture(capture);
4912                Ok(v)
4913            }
4914            Err(e) => {
4915                // Preserve sequence/group gaps without retaining failed events or evicting history.
4916                log.discard_capture(capture);
4917                Err(e)
4918            }
4919        }
4920    }
4921
4922    fn rollback_from_action_journal(
4923        &mut self,
4924        journal: &crate::engine::ActionJournal,
4925        invalidation_baseline: InvalidationBaseline,
4926    ) -> Result<(), crate::engine::EditorError> {
4927        // Invalidate first so a partial inverse failure cannot leave a changed
4928        // graph behind an apparently current schedule or lookup cache.
4929        self.invalidate_for_action_journal(
4930            journal,
4931            LoggedEditDirection::InverseReplay,
4932            invalidation_baseline,
4933        );
4934        // 1) Roll back the dependency graph structure.
4935        journal.graph.undo(&mut self.graph)?;
4936        // 2) Roll back engine row-visibility sidecar events.
4937        self.apply_inverse_row_visibility_events(&journal.graph.events);
4938        // 3) Roll back Arrow-truth overlays.
4939        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
4940        Ok(())
4941    }
4942
4943    fn rollback_from_change_events(
4944        &mut self,
4945        events: &[crate::engine::ChangeEvent],
4946        invalidation_baseline: InvalidationBaseline,
4947    ) -> Result<(), crate::engine::EditorError> {
4948        use crate::engine::ChangeEvent;
4949
4950        // Fail closed before applying inverses because replay can return after
4951        // only part of the batch has been restored.
4952        self.invalidate_for_change_events(
4953            events,
4954            LoggedEditDirection::InverseReplay,
4955            invalidation_baseline,
4956        );
4957
4958        // 1) Roll back the dependency graph.
4959        self.graph
4960            .authority_set_replay(crate::engine::authority::history::Replay::Undo);
4961        let rolled_back = (|| {
4962            let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
4963            let mut compound_stack: Vec<usize> = Vec::new();
4964            for (i, ev) in events.iter().enumerate().rev() {
4965                match ev {
4966                    ChangeEvent::CompoundEnd { depth } => {
4967                        compound_stack.push(*depth);
4968                        if let Some(description) =
4969                            crate::engine::graph::editor::change_log::compound_start_description(
4970                                i,
4971                                |j| &events[j],
4972                            )
4973                        {
4974                            editor.inverse_compound_end(description);
4975                        }
4976                    }
4977                    ChangeEvent::CompoundStart { depth, .. } => {
4978                        if compound_stack.last() == Some(depth) {
4979                            compound_stack.pop();
4980                        }
4981                        editor.apply_inverse(ev.clone())?;
4982                    }
4983                    ChangeEvent::SetRowVisibility { .. } => {
4984                        // Engine-side metadata handled after dropping graph editor borrow.
4985                    }
4986                    _ => {
4987                        editor.apply_inverse(ev.clone())?;
4988                    }
4989                }
4990            }
4991            Ok::<_, crate::engine::EditorError>(())
4992        })();
4993        self.graph
4994            .authority_set_replay(crate::engine::authority::history::Replay::Forward);
4995        rolled_back?;
4996
4997        // 2) Roll back engine row-visibility metadata.
4998        for ev in events.iter().rev() {
4999            self.apply_inverse_row_visibility_event(ev);
5000        }
5001
5002        // 3) Roll back Arrow-truth overlays mirrored from those ChangeEvents.
5003        for ev in events.iter().rev() {
5004            self.mirror_inverse_change_to_arrow(ev);
5005        }
5006
5007        Ok(())
5008    }
5009
5010    fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
5011        let sheet_id = self.graph.sheet_id(sheet)?;
5012        let coord = Coord::from_excel(row, col, true, true);
5013        let cell = CellRef::new(sheet_id, coord);
5014        let vid = self.graph.get_vertex_for_cell(&cell)?;
5015        self.graph.get_formula(vid)
5016    }
5017
5018    pub fn define_name_with_logger(
5019        &mut self,
5020        log: &mut crate::engine::ChangeLog,
5021        name: &str,
5022        definition: NamedDefinition,
5023        scope: NameScope,
5024    ) -> Result<(), crate::engine::EditorError> {
5025        self.graph
5026            .validate_define_name(name, scope)
5027            .map_err(crate::engine::EditorError::Excel)?;
5028
5029        {
5030            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5031            editor.define_name(name, definition, scope)?;
5032        }
5033        self.record_structural_change(StructuralScope::AllSheets);
5034
5035        self.mark_topology_edited();
5036
5037        Ok(())
5038    }
5039
5040    pub fn update_name_with_logger(
5041        &mut self,
5042        log: &mut crate::engine::ChangeLog,
5043        name: &str,
5044        definition: NamedDefinition,
5045        scope: NameScope,
5046    ) -> Result<(), crate::engine::EditorError> {
5047        self.graph
5048            .validate_existing_name(name, scope)
5049            .map_err(crate::engine::EditorError::Excel)?;
5050        {
5051            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5052            editor.update_name(name, definition, scope)?;
5053        }
5054        self.record_structural_change(StructuralScope::AllSheets);
5055
5056        self.mark_topology_edited();
5057
5058        Ok(())
5059    }
5060
5061    pub fn delete_name_with_logger(
5062        &mut self,
5063        log: &mut crate::engine::ChangeLog,
5064        name: &str,
5065        scope: NameScope,
5066    ) -> Result<(), crate::engine::EditorError> {
5067        self.graph
5068            .validate_existing_name(name, scope)
5069            .map_err(crate::engine::EditorError::Excel)?;
5070        {
5071            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5072            editor.delete_name(name, scope)?;
5073        }
5074        self.record_structural_change(StructuralScope::AllSheets);
5075
5076        self.mark_topology_edited();
5077
5078        Ok(())
5079    }
5080
5081    pub fn edit_with_logger<T>(
5082        &mut self,
5083        log: &mut crate::engine::ChangeLog,
5084        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5085    ) -> Result<T, crate::engine::EditorError> {
5086        let mut capture = MutationCapture::new(log.current_meta());
5087        let result = self.edit_with_capture(&mut capture, f);
5088        capture.close_compounds();
5089        match result {
5090            Ok(value) => {
5091                log.publish_capture(capture);
5092                Ok(value)
5093            }
5094            Err(error) => {
5095                log.discard_capture(capture);
5096                Err(error)
5097            }
5098        }
5099    }
5100
5101    fn edit_with_capture<T>(
5102        &mut self,
5103        capture: &mut MutationCapture,
5104        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5105    ) -> Result<T, crate::engine::EditorError> {
5106        let invalidation_baseline = self.invalidation_baseline();
5107        let start_len = capture.len();
5108        let lazy_start = capture.lazy_len();
5109
5110        // Provide a spill snapshot reader so VertexEditor can snapshot Arrow-truth spill values
5111        // (graph value cache is intentionally empty in canonical mode).
5112        struct ArrowSpillReader<'a> {
5113            sheets: &'a crate::arrow_store::SheetStore,
5114        }
5115        impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
5116            fn read_cell_value(
5117                &self,
5118                sheet: &str,
5119                row: u32,
5120                col: u32,
5121            ) -> Option<formualizer_common::LiteralValue> {
5122                use formualizer_common::LiteralValue;
5123                let asheet = self.sheets.sheet(sheet)?;
5124                let r0 = row.saturating_sub(1) as usize;
5125                let c0 = col.saturating_sub(1) as usize;
5126                let v = asheet.get_cell_value(r0, c0);
5127                if matches!(v, LiteralValue::Empty) {
5128                    None
5129                } else {
5130                    Some(v)
5131                }
5132            }
5133        }
5134
5135        let ret = {
5136            let spill_reader = ArrowSpillReader {
5137                sheets: &self.arrow_sheets,
5138            };
5139            let mut editor = crate::engine::VertexEditor::with_capture_and_spill_reader(
5140                &mut self.graph,
5141                capture,
5142                &spill_reader,
5143            );
5144            f(&mut editor)
5145        };
5146
5147        // Plain events only: run records (Program 2) stand for
5148        // `FormulaAdjusted` events, which have no forward effect here but
5149        // topology invalidation.
5150        let new_events = capture.events()[start_len..].to_vec();
5151        let new_runs = capture.lazy_len() > lazy_start;
5152        if new_events.iter().any(|event| {
5153            matches!(
5154                event,
5155                ChangeEvent::DefineName { .. }
5156                    | ChangeEvent::UpdateName { .. }
5157                    | ChangeEvent::DeleteName { .. }
5158            )
5159        }) {
5160            let all = capture.expanded_events_from(start_len, lazy_start);
5161            self.rollback_from_change_events(&all, invalidation_baseline)?;
5162            return Err(crate::engine::EditorError::TransactionUnsupported {
5163                reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
5164            });
5165        }
5166
5167        // Mirror value-impacting graph events to Arrow for forward edits.
5168        // This keeps Arrow overlays (delta + computed) consistent when edits clear/commit spills.
5169        self.clear_logged_cell_format_states(&new_events);
5170        for ev in &new_events {
5171            self.mirror_forward_change_to_arrow(ev);
5172        }
5173        for ev in &new_events {
5174            self.record_change_for_event(ev);
5175        }
5176
5177        // Atomic EngineAction calls publish one invalidation for their complete
5178        // journal at commit/rollback. Direct logged edits publish here.
5179        if self.action_depth == 0 {
5180            let mut impact =
5181                Self::classify_change_events(&new_events, LoggedEditDirection::Original);
5182            if new_runs {
5183                impact = impact.max(LoggedEditImpact::Topology);
5184            }
5185            self.apply_logged_edit_impact(impact, invalidation_baseline);
5186        }
5187
5188        Ok(ret)
5189    }
5190
5191    pub(crate) fn preflight_replay_admission(
5192        &mut self,
5193        events: &[ChangeEvent],
5194        forward: bool,
5195    ) -> Result<(), crate::engine::EditorError> {
5196        if !self.graph_admission_enabled() {
5197            return Ok(());
5198        }
5199        let mut vertex_delta = 0i128;
5200        let mut edge_delta = 0i128;
5201        let mut added_vertices = 0usize;
5202        let mut added_edges = 0usize;
5203        let mut formula_cells = BTreeSet::new();
5204        for event in events {
5205            match event {
5206                ChangeEvent::AddVertex {
5207                    formula,
5208                    coord,
5209                    sheet_id,
5210                    ..
5211                } => {
5212                    let delta = if forward { 1 } else { -1 };
5213                    vertex_delta += delta;
5214                    if forward {
5215                        added_vertices = added_vertices.saturating_add(1);
5216                        if formula.is_some() {
5217                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5218                        }
5219                    }
5220                }
5221                ChangeEvent::RemoveVertex {
5222                    old_formula,
5223                    coord,
5224                    sheet_id,
5225                    ..
5226                } => {
5227                    let delta = if forward { -1 } else { 1 };
5228                    vertex_delta += delta;
5229                    if !forward {
5230                        added_vertices = added_vertices.saturating_add(1);
5231                        if old_formula.is_some()
5232                            && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
5233                        {
5234                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5235                        }
5236                    }
5237                }
5238                ChangeEvent::EdgeAdded { .. } => {
5239                    let delta = if forward { 1 } else { -1 };
5240                    edge_delta += delta;
5241                    if forward {
5242                        added_edges = added_edges.saturating_add(1);
5243                    }
5244                }
5245                ChangeEvent::EdgeRemoved { .. } => {
5246                    let delta = if forward { -1 } else { 1 };
5247                    edge_delta += delta;
5248                    if !forward {
5249                        added_edges = added_edges.saturating_add(1);
5250                    }
5251                }
5252                ChangeEvent::SetFormula {
5253                    addr, old_formula, ..
5254                } if (forward || old_formula.is_some()) => {
5255                    formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
5256                }
5257                _ => {}
5258            }
5259        }
5260        let stats = self.graph.baseline_stats();
5261        let final_vertices = i128::try_from(stats.graph_vertex_count)
5262            .ok()
5263            .and_then(|count| count.checked_add(vertex_delta))
5264            .and_then(|count| usize::try_from(count).ok())
5265            .ok_or_else(|| {
5266                crate::engine::EditorError::Excel(
5267                    ExcelError::new(ExcelErrorKind::NImpl)
5268                        .with_message("replay vertex count overflow"),
5269                )
5270            })?;
5271        let final_edges = i128::try_from(stats.graph_edge_count)
5272            .ok()
5273            .and_then(|count| count.checked_add(edge_delta))
5274            .and_then(|count| usize::try_from(count).ok())
5275            .ok_or_else(|| {
5276                crate::engine::EditorError::Excel(
5277                    ExcelError::new(ExcelErrorKind::NImpl)
5278                        .with_message("replay edge count overflow"),
5279                )
5280            })?;
5281        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5282            final_vertices,
5283            final_edges,
5284            materialization_cells: formula_cells.len() as u64,
5285            added_vertices,
5286            added_edges,
5287        })
5288        .map_err(crate::engine::EditorError::Excel)
5289    }
5290
5291    /// Undo the last group still retained by the provided audit log.
5292    ///
5293    /// Disabled, zero-cap, and evicted history is unavailable on this index-based path. Use an
5294    /// explicit `ActionJournal` with `undo_action` when undo must be independent of audit retention.
5295    pub fn undo_logged(
5296        &mut self,
5297        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5298        log: &mut crate::engine::ChangeLog,
5299    ) -> Result<(), crate::engine::EditorError> {
5300        let pending_events = log
5301            .last_group_indices()
5302            .into_iter()
5303            .map(|index| log.events()[index].clone())
5304            .collect::<Vec<_>>();
5305        self.preflight_replay_admission(&pending_events, false)?;
5306        let invalidation_baseline = self.invalidation_baseline();
5307        // UndoEngine can fail after partially applying the batch, so publish
5308        // invalidation before replay rather than only on the success path.
5309        self.invalidate_for_change_events(
5310            &pending_events,
5311            LoggedEditDirection::InverseReplay,
5312            invalidation_baseline,
5313        );
5314        let batch = undo.undo(&mut self.graph, log)?;
5315        for item in batch.iter().rev() {
5316            self.apply_inverse_row_visibility_event(&item.event);
5317            self.apply_inverse_staged_formula_event(&item.event);
5318        }
5319        if !batch.is_empty() {
5320            let events = batch
5321                .iter()
5322                .map(|item| item.event.clone())
5323                .collect::<Vec<_>>();
5324            self.clear_logged_cell_format_states(&events);
5325        }
5326        self.mirror_undo_batch_to_arrow(&batch);
5327        if !batch.is_empty() {
5328            for item in &batch {
5329                self.record_change_for_event(&item.event);
5330            }
5331        }
5332        crate::engine::trace::fz_event!(
5333            tracing::Level::INFO,
5334            "history",
5335            "history.replay",
5336            op = "undo",
5337            events_replayed = batch.len(),
5338            ownership_resyncs = batch.len()
5339        );
5340        Ok(())
5341    }
5342
5343    pub fn redo_logged(
5344        &mut self,
5345        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5346        log: &mut crate::engine::ChangeLog,
5347    ) -> Result<(), crate::engine::EditorError> {
5348        let pending_events = undo.pending_redo_events();
5349        self.preflight_replay_admission(&pending_events, true)?;
5350        let invalidation_baseline = self.invalidation_baseline();
5351        self.invalidate_for_change_events(
5352            &pending_events,
5353            LoggedEditDirection::ForwardReplay,
5354            invalidation_baseline,
5355        );
5356        let batch = undo.redo(&mut self.graph, log)?;
5357        for item in &batch {
5358            self.apply_forward_row_visibility_event(&item.event);
5359            self.apply_forward_staged_formula_event(&item.event);
5360        }
5361        if !batch.is_empty() {
5362            let events = batch
5363                .iter()
5364                .map(|item| item.event.clone())
5365                .collect::<Vec<_>>();
5366            self.clear_logged_cell_format_states(&events);
5367        }
5368        self.mirror_redo_batch_to_arrow(&batch);
5369        if !batch.is_empty() {
5370            for item in &batch {
5371                self.record_change_for_event(&item.event);
5372            }
5373        }
5374        crate::engine::trace::fz_event!(
5375            tracing::Level::INFO,
5376            "history",
5377            "history.replay",
5378            op = "redo",
5379            events_replayed = batch.len(),
5380            ownership_resyncs = batch.len()
5381        );
5382        Ok(())
5383    }
5384
5385    /// Undo the last committed atomic action using the journal stack.
5386    ///
5387    /// This path does not require a `ChangeLog`.
5388    pub fn undo_action(
5389        &mut self,
5390        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5391    ) -> Result<(), crate::engine::EditorError> {
5392        let Some(journal) = undo.pop_undo_action() else {
5393            return Ok(());
5394        };
5395        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5396            undo.push_done_action(journal);
5397            return Err(error);
5398        }
5399        let invalidation_baseline = self.invalidation_baseline();
5400
5401        self.invalidate_for_action_journal(
5402            &journal,
5403            LoggedEditDirection::InverseReplay,
5404            invalidation_baseline,
5405        );
5406        journal.graph.undo(&mut self.graph)?;
5407        self.apply_inverse_row_visibility_events(&journal.graph.events);
5408        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
5409        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5410            for event in &journal.graph.events {
5411                self.record_change_for_event(event);
5412            }
5413        }
5414
5415        #[cfg(feature = "tracing")]
5416        let events_replayed = journal.graph.events.len();
5417        crate::engine::trace::fz_event!(
5418            tracing::Level::INFO,
5419            "history",
5420            "history.replay",
5421            op = "undo",
5422            events_replayed,
5423            ownership_resyncs = events_replayed
5424        );
5425        undo.push_redo_action(journal);
5426        Ok(())
5427    }
5428
5429    /// Redo the last undone atomic action using the journal stack.
5430    ///
5431    /// This path does not require a `ChangeLog`.
5432    pub fn redo_action(
5433        &mut self,
5434        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5435    ) -> Result<(), crate::engine::EditorError> {
5436        let Some(journal) = undo.pop_redo_action() else {
5437            return Ok(());
5438        };
5439        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5440            undo.push_redo_action(journal);
5441            return Err(error);
5442        }
5443        let invalidation_baseline = self.invalidation_baseline();
5444        self.invalidate_for_action_journal(
5445            &journal,
5446            LoggedEditDirection::ForwardReplay,
5447            invalidation_baseline,
5448        );
5449        journal.graph.redo(&mut self.graph)?;
5450        self.apply_forward_row_visibility_events(&journal.graph.events);
5451        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ false);
5452        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5453            for event in &journal.graph.events {
5454                self.record_change_for_event(event);
5455            }
5456        }
5457
5458        #[cfg(feature = "tracing")]
5459        let events_replayed = journal.graph.events.len();
5460        crate::engine::trace::fz_event!(
5461            tracing::Level::INFO,
5462            "history",
5463            "history.replay",
5464            op = "redo",
5465            events_replayed,
5466            ownership_resyncs = events_replayed
5467        );
5468        undo.push_done_action(journal);
5469        Ok(())
5470    }
5471
5472    fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5473        let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5474        // Coord stores 0-based indices.
5475        let row = addr.coord.row() + 1;
5476        let col = addr.coord.col() + 1;
5477        (sheet, row, col)
5478    }
5479
5480    fn mirror_undo_batch_to_arrow(
5481        &mut self,
5482        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5483    ) {
5484        // Undo applies inverses in reverse order.
5485        for item in batch.iter().rev() {
5486            self.mirror_inverse_change_to_arrow(&item.event);
5487        }
5488    }
5489
5490    fn mirror_redo_batch_to_arrow(
5491        &mut self,
5492        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5493    ) {
5494        // Redo applies events in forward order.
5495        for item in batch.iter() {
5496            self.mirror_forward_change_to_arrow(&item.event);
5497        }
5498    }
5499
5500    fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5501        use crate::engine::ChangeEvent;
5502        use formualizer_common::LiteralValue;
5503
5504        match ev {
5505            ChangeEvent::SetValue {
5506                addr,
5507                old_value,
5508                old_formula,
5509                ..
5510            } => {
5511                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5512                if old_formula.is_some() {
5513                    self.clear_delta_overlay_cell(&sheet, row, col);
5514                } else {
5515                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5516                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5517                }
5518            }
5519            ChangeEvent::SetFormula {
5520                addr,
5521                old_value,
5522                old_formula,
5523                ..
5524            } => {
5525                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5526                if old_formula.is_some() {
5527                    self.clear_delta_overlay_cell(&sheet, row, col);
5528                } else {
5529                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5530                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5531                }
5532            }
5533            ChangeEvent::SpillCommitted { old, new, .. } => {
5534                // Inverse: restore `old` (or clear if none).
5535                self.mirror_spill_snapshot(new, /*clear_only=*/ true);
5536                if let Some(snap) = old {
5537                    self.mirror_spill_snapshot(snap, /*clear_only=*/ false);
5538                }
5539            }
5540            ChangeEvent::SpillCleared { old, .. } => {
5541                // Inverse: restore prior spill.
5542                self.mirror_spill_snapshot(old, /*clear_only=*/ false);
5543            }
5544            ChangeEvent::SetRowVisibility { .. } => {
5545                // Engine-side metadata only; no Arrow overlay effect.
5546            }
5547            _ => {}
5548        }
5549    }
5550
5551    fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5552        use crate::engine::ChangeEvent;
5553
5554        match ev {
5555            ChangeEvent::SetValue { addr, new, .. } => {
5556                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5557                self.mirror_value_to_overlay(&sheet, row, col, new);
5558            }
5559            ChangeEvent::SetFormula { addr, .. } => {
5560                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5561                self.clear_delta_overlay_cell(&sheet, row, col);
5562                // Keep any computed overlay for this cell as-is; it will be recomputed on demand.
5563            }
5564            ChangeEvent::SpillCommitted { old, new, .. } => {
5565                if let Some(snap) = old {
5566                    self.mirror_spill_snapshot(snap, /*clear_only=*/ true);
5567                }
5568                self.mirror_spill_snapshot(new, /*clear_only=*/ false);
5569            }
5570            ChangeEvent::SpillCleared { old, .. } => {
5571                self.mirror_spill_snapshot(old, /*clear_only=*/ true);
5572            }
5573            ChangeEvent::SetRowVisibility { .. } => {
5574                // Engine-side metadata only; no Arrow overlay effect.
5575            }
5576            _ => {
5577                // Other graph structural operations do not have direct value effects in Arrow.
5578            }
5579        }
5580    }
5581
5582    fn mirror_spill_snapshot(
5583        &mut self,
5584        snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5585        clear_only: bool,
5586    ) {
5587        use formualizer_common::LiteralValue;
5588
5589        let mut i = 0usize;
5590        for row in &snap.values {
5591            for v in row {
5592                if let Some(cell) = snap.target_cells.get(i) {
5593                    let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5594                    let out = if clear_only {
5595                        LiteralValue::Empty
5596                    } else {
5597                        v.clone()
5598                    };
5599                    self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5600                }
5601                i += 1;
5602            }
5603        }
5604        // If target_cells is longer than values (should not happen), clear remaining cells.
5605        if clear_only {
5606            for cell in snap.target_cells.iter().skip(i) {
5607                let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5608                self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5609            }
5610        }
5611    }
5612
5613    pub fn set_default_sheet_by_name(&mut self, name: &str) {
5614        self.graph.set_default_sheet_by_name(name);
5615    }
5616
5617    pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5618        self.graph.set_default_sheet_by_id(id);
5619    }
5620
5621    pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5622        self.graph.set_sheet_index_mode(mode);
5623    }
5624
5625    #[cfg(feature = "test-support")]
5626    #[doc(hidden)]
5627    pub fn mark_all_formulas_dirty_for_test(&mut self) {
5628        self.mark_all_formula_vertices_dirty();
5629    }
5630
5631    #[cfg(feature = "test-support")]
5632    #[doc(hidden)]
5633    pub fn take_criteria_mask_work_for_test() -> (usize, usize) {
5634        criteria_mask_test_hooks::take_mask_work()
5635    }
5636
5637    #[cfg(feature = "test-support")]
5638    #[doc(hidden)]
5639    pub fn lookup_index_cache_report_for_test(&self) -> LookupIndexCacheReport {
5640        self.lookup_index_cache.report()
5641    }
5642
5643    #[cfg(any(test, feature = "benchmark_internal"))]
5644    #[doc(hidden)]
5645    pub fn reset_recalc_reuse_probe(&mut self) {
5646        *self.recalc_reuse_probe.get_mut().unwrap() = RecalcReuseProbe::default();
5647    }
5648
5649    #[cfg(any(test, feature = "benchmark_internal"))]
5650    #[doc(hidden)]
5651    pub fn recalc_reuse_probe(&self) -> RecalcReuseProbe {
5652        let mut probe = self.recalc_reuse_probe.lock().unwrap().clone();
5653        if let Some(cached) = self.cached_static_schedule.as_ref() {
5654            let entry_bytes = |e: &CachedScheduleEntry| {
5655                std::mem::size_of::<CachedScheduleEntry>()
5656                    + e.candidate_vertices.heap_bytes()
5657                    + std::mem::size_of::<crate::engine::Schedule>()
5658                    + 2 * std::mem::size_of::<usize>()
5659                    + schedule_probe_retained_bytes(&e.schedule)
5660            };
5661            probe.schedule_retained_bytes = entry_bytes(cached)
5662                + self.recent_schedules.iter().map(entry_bytes).sum::<usize>()
5663                + self.base_schedule.as_ref().map_or(0, entry_bytes)
5664                + self.recent_schedules.capacity() * std::mem::size_of::<CachedScheduleEntry>();
5665        }
5666        probe
5667    }
5668
5669    #[cfg(test)]
5670    pub(crate) fn cached_static_schedule_for_test(&self) -> Option<Arc<crate::engine::Schedule>> {
5671        self.cached_static_schedule
5672            .as_ref()
5673            .map(|cached| Arc::clone(&cached.schedule))
5674    }
5675
5676    fn clear_cached_static_schedule(&mut self) {
5677        self.cached_static_schedule = None;
5678        self.recent_schedules.clear();
5679        self.base_schedule = None;
5680    }
5681
5682    /// Keep a replaced schedule among the recent ones when it is still
5683    /// current and small; drop stale ones and the oldest beyond the bounds.
5684    fn retain_recent_schedule(&mut self, entry: CachedScheduleEntry) {
5685        const RECENT_SCHEDULES: usize = 8;
5686        const RECENT_SCHEDULE_VERTICES: usize = 65_536;
5687        let revision = self.schedule_cache_authority_revision();
5688        let epoch = self.topology_epoch;
5689        self.recent_schedules
5690            .retain(|e| e.topology_epoch == epoch && e.authority_revision == revision);
5691        let current =
5692            |e: &CachedScheduleEntry| e.topology_epoch == epoch && e.authority_revision == revision;
5693        if self.base_schedule.as_ref().is_some_and(|b| !current(b)) {
5694            self.base_schedule = None;
5695        }
5696        // The largest current schedule becomes the base; a replaced base
5697        // may still join the recent ones.
5698        let mut entry = entry;
5699        if current(&entry)
5700            && entry.candidate_vertices.len() > BASE_SCHEDULE_MIN_VERTICES
5701            && self
5702                .base_schedule
5703                .as_ref()
5704                .is_none_or(|b| entry.candidate_vertices.len() > b.candidate_vertices.len())
5705        {
5706            match self.base_schedule.replace(entry) {
5707                Some(previous) => entry = previous,
5708                None => return,
5709            }
5710        }
5711        if entry.topology_epoch != epoch
5712            || entry.authority_revision != revision
5713            || entry.candidate_vertices.len() > RECENT_SCHEDULE_VERTICES / 4
5714        {
5715            return;
5716        }
5717        self.recent_schedules.push(entry);
5718        let mut total: usize = self
5719            .recent_schedules
5720            .iter()
5721            .map(|e| e.candidate_vertices.len())
5722            .sum();
5723        while self.recent_schedules.len() > RECENT_SCHEDULES || total > RECENT_SCHEDULE_VERTICES {
5724            let oldest = self.recent_schedules.remove(0);
5725            total -= oldest.candidate_vertices.len();
5726        }
5727    }
5728
5729    fn invalidation_baseline(&self) -> InvalidationBaseline {
5730        InvalidationBaseline {
5731            snapshot_id: self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed),
5732            topology_epoch: self.topology_epoch,
5733        }
5734    }
5735
5736    fn classify_change_events(
5737        events: &[crate::engine::ChangeEvent],
5738        direction: LoggedEditDirection,
5739    ) -> LoggedEditImpact {
5740        use crate::engine::ChangeEvent;
5741
5742        events
5743            .iter()
5744            .map(|event| match event {
5745                ChangeEvent::CompoundStart { .. } | ChangeEvent::CompoundEnd { .. } => {
5746                    LoggedEditImpact::NoOp
5747                }
5748                ChangeEvent::SetRowVisibility { .. }
5749                | ChangeEvent::SetValue {
5750                    old_formula: None,
5751                    old_value: Some(_),
5752                    ..
5753                } => LoggedEditImpact::DataOnly,
5754                // Original canonical writes can lack an Arrow old value while
5755                // updating an existing placeholder. Inverse replay actually
5756                // removes that vertex; a later redo recreates it.
5757                ChangeEvent::SetValue {
5758                    old_formula: None,
5759                    old_value: None,
5760                    ..
5761                } if direction == LoggedEditDirection::Original => LoggedEditImpact::DataOnly,
5762                ChangeEvent::SetValue { .. }
5763                | ChangeEvent::SetFormula { .. }
5764                | ChangeEvent::AddVertex { .. }
5765                | ChangeEvent::RemoveVertex { .. }
5766                | ChangeEvent::VertexMoved { .. }
5767                | ChangeEvent::FormulaAdjusted { .. }
5768                | ChangeEvent::NamedRangeAdjusted { .. }
5769                | ChangeEvent::EdgeAdded { .. }
5770                | ChangeEvent::EdgeRemoved { .. }
5771                | ChangeEvent::DefineName { .. }
5772                | ChangeEvent::UpdateName { .. }
5773                | ChangeEvent::DeleteName { .. }
5774                | ChangeEvent::SpillCommitted { .. }
5775                | ChangeEvent::SpillCleared { .. }
5776                | ChangeEvent::StagedFormulaCellChanged { .. } => LoggedEditImpact::Topology,
5777            })
5778            .max()
5779            .unwrap_or(LoggedEditImpact::NoOp)
5780    }
5781
5782    fn classify_arrow_undo(arrow: &crate::engine::ArrowUndoBatch) -> LoggedEditImpact {
5783        use crate::engine::ArrowOp;
5784
5785        arrow
5786            .ops
5787            .iter()
5788            .map(|op| match op {
5789                ArrowOp::SetDeltaCell { .. } | ArrowOp::SetComputedCell { .. } => {
5790                    LoggedEditImpact::DataOnly
5791                }
5792                ArrowOp::RestoreComputedRect { .. }
5793                | ArrowOp::InsertRows { .. }
5794                | ArrowOp::InsertCols { .. } => LoggedEditImpact::Topology,
5795            })
5796            .max()
5797            .unwrap_or(LoggedEditImpact::NoOp)
5798    }
5799
5800    fn apply_logged_edit_impact(
5801        &mut self,
5802        impact: LoggedEditImpact,
5803        baseline: InvalidationBaseline,
5804    ) {
5805        match impact {
5806            LoggedEditImpact::NoOp => {}
5807            LoggedEditImpact::DataOnly => {
5808                if self.topology_epoch == baseline.topology_epoch
5809                    && self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
5810                        == baseline.snapshot_id
5811                {
5812                    self.mark_data_edited();
5813                }
5814            }
5815            LoggedEditImpact::Topology => {
5816                // Some structural entry points already publish topology
5817                // invalidation. Do not bump the same batch twice.
5818                if self.topology_epoch == baseline.topology_epoch {
5819                    self.mark_topology_edited();
5820                }
5821            }
5822        }
5823    }
5824
5825    fn invalidate_for_change_events(
5826        &mut self,
5827        events: &[crate::engine::ChangeEvent],
5828        direction: LoggedEditDirection,
5829        baseline: InvalidationBaseline,
5830    ) {
5831        self.apply_logged_edit_impact(Self::classify_change_events(events, direction), baseline);
5832    }
5833
5834    fn invalidate_for_action_journal(
5835        &mut self,
5836        journal: &crate::engine::ActionJournal,
5837        direction: LoggedEditDirection,
5838        baseline: InvalidationBaseline,
5839    ) {
5840        let impact = Self::classify_change_events(&journal.graph.events, direction)
5841            .max(Self::classify_arrow_undo(&journal.arrow));
5842        self.apply_logged_edit_impact(impact, baseline);
5843    }
5844
5845    /// Mark data edited: bump snapshot and set edited flag.
5846    /// Value-only edits keep the stable-topology schedule cache alive.
5847    pub fn mark_data_edited(&mut self) {
5848        self.lookup_index_cache.clear();
5849        self.snapshot_id
5850            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5851        self.has_edited = true;
5852    }
5853
5854    /// Mark a topology-changing edit: bump snapshot + topology epoch and invalidate cached schedules.
5855    pub fn mark_topology_edited(&mut self) {
5856        self.lookup_index_cache.clear();
5857        self.snapshot_id
5858            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5859        self.topology_epoch = self.topology_epoch.wrapping_add(1);
5860        self.graph.bump_topology_revision();
5861        self.clear_cached_static_schedule();
5862        if let Some(ledger) = self.active_resource_ledger.as_mut() {
5863            let released = ledger.account_mixed_cache(0);
5864            debug_assert!(released.is_ok());
5865        }
5866        self.has_edited = true;
5867        // Eager sync at a topology edit, so read-only (`&self`) plans and
5868        // inspection see a current authority (not during a load or an open
5869        // structural capture).
5870        self.graph.authority_sync_eager();
5871    }
5872
5873    fn mark_all_formula_vertices_dirty(&mut self) {
5874        let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
5875        for vertex in vertices {
5876            self.graph.mark_vertex_dirty(vertex);
5877        }
5878    }
5879
5880    fn mark_moved_formula_vertices_dirty(
5881        &mut self,
5882        summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
5883    ) {
5884        for vertex in &summary.vertices_moved {
5885            if self.graph.has_formula(*vertex) {
5886                self.graph.mark_vertex_dirty(*vertex);
5887            }
5888        }
5889    }
5890
5891    /// Access Arrow sheet store (read-only)
5892    pub fn sheet_store(&self) -> &SheetStore {
5893        &self.arrow_sheets
5894    }
5895
5896    /// True when any sheet carries manual/filter row-visibility state.
5897    /// Used by load-time freshness checks (see `Engine::adopt_file_sheets`).
5898    pub(crate) fn has_row_visibility_state(&self) -> bool {
5899        !self.row_visibility.is_empty()
5900    }
5901
5902    /// Access Arrow sheet store (mutable)
5903    pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
5904        &mut self.arrow_sheets
5905    }
5906
5907    pub fn has_staged_formulas(&self) -> bool {
5908        !self.staged_formulas.is_empty()
5909    }
5910
5911    pub fn staged_formula_count(&self) -> usize {
5912        self.staged_formulas.values().map(StagedSheet::len).sum()
5913    }
5914
5915    /// Stage a formula text instead of inserting into the graph (used when deferring is enabled).
5916    pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
5917        self.staged_formulas
5918            .entry(sheet.to_string())
5919            .or_default()
5920            .stage(row, col, text);
5921        self.staged_formula_index.stage(sheet, row, col);
5922        if let Some(sheet) = self.graph.sheet_id(sheet) {
5923            self.invalidate_pending_spills(StructuralScope::Cell {
5924                sheet,
5925                row: row.saturating_sub(1),
5926                col: col.saturating_sub(1),
5927            });
5928        }
5929    }
5930
5931    fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
5932        for (row, col, _) in &staged.entries {
5933            self.staged_formula_index.remove(sheet, *row, *col);
5934        }
5935        if staged.deferred_package.is_some() {
5936            self.staged_formula_index.set_package(sheet, None);
5937        }
5938    }
5939
5940    fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
5941        self.staged_formulas.insert(sheet, staged);
5942    }
5943
5944    #[doc(hidden)]
5945    pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
5946        SourceFormulaIngress { engine: self }
5947    }
5948
5949    #[doc(hidden)]
5950    /// Test-only fault-injection seam. Not part of the supported API; it exists so
5951    /// integration tests in sibling crates can fail a commit at an exact point.
5952    #[doc(hidden)]
5953    #[cfg(any(test, feature = "test-support"))]
5954    pub fn set_before_prepared_span_commit_hook(
5955        &mut self,
5956        hook: impl FnOnce() + Send + Sync + 'static,
5957    ) {
5958        self.before_prepared_span_commit_hook = Some(Box::new(hook));
5959    }
5960
5961    #[doc(hidden)]
5962    /// Test-only fault-injection seam, matching `set_after_eager_proposal_commit_hook`.
5963    #[cfg(test)]
5964    pub(crate) fn set_before_target_preparation_commit_hook(
5965        &mut self,
5966        hook: impl FnOnce() + Send + Sync + 'static,
5967    ) {
5968        self.before_target_preparation_commit_hook = Some(Box::new(hook));
5969    }
5970
5971    fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
5972        if let Ok(replay) = package.replay.lock()
5973            && let Some(footprint) = replay.selection_cache_footprint()
5974            && !self
5975                .source_cache_footprints
5976                .iter()
5977                .any(|known| known.ptr_eq(&footprint))
5978        {
5979            self.source_cache_footprints.push(footprint);
5980        }
5981        let sheet = package.sheet_name.clone();
5982        let staged = self.staged_formulas.entry(sheet.clone()).or_default();
5983        debug_assert!(staged.deferred_package.is_none());
5984        staged.deferred_package = Some(package);
5985        staged.reconcile_attached_deferred_package();
5986        self.staged_formula_index
5987            .set_package(&sheet, staged.deferred_package.as_ref());
5988        if let Some(sheet_id) = self.graph.sheet_id(&sheet) {
5989            let package = self
5990                .staged_formulas
5991                .get(&sheet)
5992                .and_then(|staged| staged.deferred_package.as_ref());
5993            for &(vertex, anchor, region) in &self.blocked_pending_spills {
5994                if anchor.sheet_id == sheet_id
5995                    && self.graph.vertex_exists(vertex)
5996                    && self.graph.get_cell_ref(vertex) == Some(anchor)
5997                    && self.staged_formula_index.package_occupies_spill(
5998                        &sheet,
5999                        (anchor.coord.row() + 1, anchor.coord.col() + 1),
6000                        (
6001                            region.rows.query_bounds().1 + 1,
6002                            region.cols.query_bounds().1 + 1,
6003                        ),
6004                        |point| package.is_some_and(|package| package.suppressed.contains(&point)),
6005                    )
6006                {
6007                    self.graph.mark_vertex_dirty(vertex);
6008                }
6009            }
6010        }
6011    }
6012
6013    pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
6014        let mut removed = None;
6015        let mut remove_sheet = false;
6016        let mut had_package = false;
6017        if let Some(entries) = self.staged_formulas.get_mut(sheet) {
6018            had_package = entries.deferred_package.is_some();
6019            removed = entries.remove(row, col);
6020            remove_sheet = entries.is_empty();
6021        }
6022        let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
6023        if !ordinary_removed && had_package {
6024            self.staged_formula_index.touch_package(sheet);
6025        }
6026        if remove_sheet {
6027            self.staged_formulas.remove(sheet);
6028            self.staged_formula_index.set_package(sheet, None);
6029        }
6030        if (ordinary_removed || had_package)
6031            && let Some(sheet) = self.graph.sheet_id(sheet)
6032        {
6033            self.invalidate_pending_spills(StructuralScope::Cell {
6034                sheet,
6035                row: row.saturating_sub(1),
6036                col: col.saturating_sub(1),
6037            });
6038        }
6039        removed
6040    }
6041
6042    pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
6043        if self.staged_formulas.remove(sheet).is_some() {
6044            self.staged_formula_index.clear_sheet(sheet);
6045            if let Some(sheet_id) = self.graph.sheet_id(sheet) {
6046                self.invalidate_pending_spills(StructuralScope::Sheet(sheet_id));
6047            }
6048        }
6049    }
6050
6051    pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
6052        let Some(entries) = self.staged_formulas.remove(old) else {
6053            return;
6054        };
6055        self.staged_formula_index.clear_sheet(old);
6056        let (formulas, mut package) = entries.into_parts();
6057        for (row, col, text) in formulas {
6058            self.stage_formula_text(new, row, col, text);
6059        }
6060        if let Some(package) = package.as_mut() {
6061            package.sheet_name = new.to_string();
6062        }
6063        if let Some(package) = package {
6064            self.stage_deferred_formula_package(package);
6065        }
6066    }
6067
6068    /// Get a staged formula text for a given cell if present (cloned).
6069    pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
6070        self.staged_formulas
6071            .get(sheet)
6072            .and_then(|v| v.get(row, col))
6073    }
6074
6075    pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
6076        &self.formula_parse_diagnostics
6077    }
6078
6079    pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
6080        std::mem::take(&mut self.formula_parse_diagnostics)
6081    }
6082
6083    pub fn clear_formula_parse_diagnostics(&mut self) {
6084        self.formula_parse_diagnostics.clear();
6085    }
6086
6087    pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
6088        self.last_formula_ingest_report.as_ref()
6089    }
6090
6091    pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
6092        &self.formula_ingest_report_total
6093    }
6094
6095    #[cfg(test)]
6096    pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
6097        &mut self,
6098        hook: impl FnOnce() + Send + Sync + 'static,
6099    ) {
6100        self.before_target_planning_snapshot_hook = Some(Box::new(hook));
6101    }
6102
6103    #[cfg(test)]
6104    pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
6105        self.inject_target_semantic_stale_once_for_test = true;
6106    }
6107
6108    #[cfg(test)]
6109    pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
6110        self.force_virtual_dep_changes_remaining_for_test = rounds;
6111    }
6112
6113    #[cfg(test)]
6114    pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
6115        self.fail_evaluation_commit_preflight_once_for_test = true;
6116    }
6117
6118    #[cfg(test)]
6119    pub(crate) fn set_target_preparation_fault_for_test(
6120        &mut self,
6121        fault: crate::engine::target_preparation::TargetPreparationFault,
6122    ) {
6123        self.target_preparation_fault_for_test = Some(fault);
6124    }
6125
6126    #[cfg(test)]
6127    pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
6128        self.staged_formula_index.revision()
6129    }
6130
6131    #[cfg(test)]
6132    pub(crate) fn deferred_package_for_test(
6133        &self,
6134        sheet: &str,
6135    ) -> &crate::engine::DeferredFormulaPackage {
6136        self.staged_formulas
6137            .get(sheet)
6138            .unwrap()
6139            .deferred_package
6140            .as_ref()
6141            .unwrap()
6142    }
6143
6144    #[cfg(test)]
6145    pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
6146        let ordinary_storage = self
6147            .staged_formulas
6148            .values()
6149            .map(|sheet| sheet.entries.len())
6150            .sum::<usize>();
6151        let package_storage = self
6152            .staged_formulas
6153            .values()
6154            .filter(|sheet| sheet.deferred_package.is_some())
6155            .count();
6156        ordinary_storage == self.staged_formula_index.ordinary_count()
6157            && package_storage == self.staged_formula_index.package_count()
6158            && self.staged_formulas.iter().all(|(name, sheet)| {
6159                sheet.entries.iter().all(|(row, col, _)| {
6160                    let leases = self
6161                        .staged_formula_index
6162                        .leases_in_region(name, *row, *col, *row, *col);
6163                    leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
6164                })
6165            })
6166    }
6167
6168    #[cfg(test)]
6169    pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
6170        self.evaluation_request_begin_count_for_test
6171    }
6172
6173    #[cfg(test)]
6174    pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
6175        &mut self,
6176        hook: impl FnOnce() + Send + Sync + 'static,
6177    ) {
6178        self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
6179    }
6180
6181    #[cfg(test)]
6182    pub(crate) fn set_after_eager_proposal_commit_hook(
6183        &mut self,
6184        hook: impl FnOnce() + Send + Sync + 'static,
6185    ) {
6186        self.after_eager_proposal_commit_hook = Some(Box::new(hook));
6187    }
6188
6189    #[cfg(test)]
6190    pub(crate) fn topology_epoch_for_test(&self) -> u64 {
6191        self.topology_epoch
6192    }
6193
6194    #[cfg(test)]
6195    pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
6196        self.graph.topology_revision()
6197    }
6198
6199    fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
6200        self.formula_ingest_report_total.mode = report.mode;
6201        self.formula_ingest_report_total.accumulate(&report);
6202        self.last_formula_ingest_report = Some(report);
6203    }
6204
6205    fn collect_planning_function_requests(
6206        ast: &ASTNode,
6207        requests: &mut Vec<(String, String, usize)>,
6208    ) {
6209        match &ast.node_type {
6210            ASTNodeType::Function { name, args } => {
6211                requests.push((String::new(), name.clone(), args.len()));
6212                for arg in args {
6213                    Self::collect_planning_function_requests(arg, requests);
6214                }
6215            }
6216            ASTNodeType::BinaryOp { left, right, .. } => {
6217                Self::collect_planning_function_requests(left, requests);
6218                Self::collect_planning_function_requests(right, requests);
6219            }
6220            ASTNodeType::UnaryOp { expr, .. } => {
6221                Self::collect_planning_function_requests(expr, requests);
6222            }
6223            ASTNodeType::Call { callee, args } => {
6224                Self::collect_planning_function_requests(callee, requests);
6225                for arg in args {
6226                    Self::collect_planning_function_requests(arg, requests);
6227                }
6228            }
6229            ASTNodeType::Array(rows) => {
6230                for cell in rows.iter().flatten() {
6231                    Self::collect_planning_function_requests(cell, requests);
6232                }
6233            }
6234            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6235        }
6236    }
6237
6238    fn prepared_function_semantics_changed(
6239        &self,
6240        preparation: &crate::engine::FormulaCompressedPreparation,
6241        guard: &crate::function_registry::SemanticEpochReadGuard,
6242    ) -> bool {
6243        if preparation.function_semantic_epoch == guard.epoch() {
6244            return false;
6245        }
6246
6247        guard
6248            .semantic_changes_affect_requests_since(preparation.function_semantic_epoch, Vec::new())
6249    }
6250
6251    pub(crate) fn prepare_source_formula_families(
6252        &mut self,
6253        sheet_name: &str,
6254        families: &[crate::engine::SourceFormulaFamily],
6255    ) -> crate::engine::FormulaCompressedPreparation {
6256        crate::engine::FormulaCompressedPreparation {
6257            engine_token: Arc::clone(&self.source_formula_token),
6258            function_semantic_epoch: crate::function_registry::semantic_epoch(),
6259            function_provider_revision: None,
6260            function_semantics_used: false,
6261            sheet_name: Arc::from(sheet_name),
6262            rejected: BTreeMap::new(),
6263            eager_replay: Vec::new(),
6264            preparation_spool_replays: 0,
6265            clean_rejected_anchor_counts: [0; 3],
6266            fragmented_rejected_anchor_counts: [0; 3],
6267            exact_replay: None,
6268            replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6269        }
6270    }
6271
6272    fn prepare_source_formula_proposals(
6273        &mut self,
6274        sheet_name: &str,
6275        families: &[crate::engine::SourceFormulaFamily],
6276        authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6277        replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6278        formula_record_count: u64,
6279        replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6280        suppressed: &BTreeSet<(u32, u32)>,
6281        consumed: &FxHashSet<(crate::engine::SourceFamilyId, crate::engine::SourceCoord)>,
6282        consumed_engine: Option<&Arc<()>>,
6283    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
6284        let mut preparation = self.prepare_source_formula_families(sheet_name, families);
6285        preparation.exact_replay = Some(Arc::clone(&replay));
6286        preparation
6287            .replay_disposition
6288            .register_consumed_members(consumed_engine, consumed.iter().copied());
6289        if !preparation
6290            .replay_disposition
6291            .consumed_engine_matches(&self.source_formula_token)
6292        {
6293            return Err(ExcelError::new(ExcelErrorKind::Value)
6294                .with_message("ResidualConsumedEngineMismatch"));
6295        }
6296        preparation
6297            .replay_disposition
6298            .extend_suppressed_excel_coords(suppressed.iter().copied());
6299        Ok(preparation)
6300    }
6301
6302    fn formula_batch_from_exact_replay(
6303        &mut self,
6304        sheet_name: &str,
6305        replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
6306    ) -> Result<FormulaIngestBatch, ExcelError> {
6307        let mut cache = rustc_hash::FxHashMap::default();
6308        let mut formulas = Vec::new();
6309        for record in replayed {
6310            let key = if record.text.starts_with('=') {
6311                record.text
6312            } else {
6313                format!("={}", record.text)
6314            };
6315            let ast_id = if let Some(cached) = cache.get(&key) {
6316                *cached
6317            } else {
6318                let parsed = match formualizer_parse::parser::parse(&key) {
6319                    Ok(parsed) => parsed,
6320                    Err(error) => {
6321                        let Some(parsed) = self.handle_formula_parse_error(
6322                            sheet_name,
6323                            record.row,
6324                            record.col,
6325                            &key,
6326                            error.to_string(),
6327                        )?
6328                        else {
6329                            continue;
6330                        };
6331                        parsed
6332                    }
6333                };
6334                let ast_id = self.intern_formula_ast(&parsed);
6335                cache.insert(key.clone(), ast_id);
6336                ast_id
6337            };
6338            formulas.push(
6339                FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
6340                    .with_source_proof(record.source_order, record.family, record.partition_owner),
6341            );
6342        }
6343        Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
6344    }
6345
6346    fn prepare_target_combined_legacy_graph(
6347        &self,
6348        packages: &[PreparedTargetSourcePackage],
6349        ordinary: &[PreparedOrdinaryStagedFormula],
6350    ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
6351        let mut planned_by_coord = BTreeMap::new();
6352        for package in packages {
6353            for (row, col, ast_id, plan) in &package.legacy {
6354                planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
6355            }
6356        }
6357        for formula in ordinary {
6358            if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
6359                planned_by_coord.insert(
6360                    (formula.sheet_id, formula.lease.row, formula.lease.col),
6361                    (ast_id, plan),
6362                );
6363            }
6364        }
6365        let planned = planned_by_coord
6366            .into_iter()
6367            .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
6368            .collect::<Vec<_>>();
6369        let formula_count = planned.len();
6370        let graph = self
6371            .graph
6372            .prepare_legacy_graph_plan_multi_sheet(planned)
6373            .map_err(|error| {
6374                ExcelError::new(ExcelErrorKind::Value)
6375                    .with_message(format!("target graph preparation failed: {error}"))
6376            })?;
6377        Ok((graph, formula_count))
6378    }
6379
6380    fn replay_target_coordinates(
6381        &mut self,
6382        replay: &Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6383        coordinates: &[(u32, u32)],
6384        deadline: Option<std::time::Instant>,
6385        scratch: &mut u64,
6386    ) -> Result<Option<Vec<crate::engine::DeferredReplayFormula>>, ExcelError> {
6387        if coordinates.is_empty() {
6388            return Ok(Some(Vec::new()));
6389        }
6390        let mut guard = replay.lock().map_err(|_| {
6391            ExcelError::new(ExcelErrorKind::Value)
6392                .with_message("deferred formula spool lock poisoned")
6393        })?;
6394        let retained = guard.selection_cache_footprint().is_some();
6395        let records = guard.replay_selected_exact(coordinates, &mut |work, bytes| {
6396            self.target_preparation_checkpoint(deadline, work)?;
6397            if retained && let Some(ledger) = self.active_resource_ledger.as_mut() {
6398                ledger
6399                    .reserve_retained(bytes)
6400                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
6401                self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
6402            }
6403            self.reserve_graph_source_scratch(bytes)?;
6404            *scratch = scratch.saturating_add(bytes);
6405            Ok(())
6406        });
6407        drop(guard);
6408        let reconciled = self.reconcile_source_cache_footprints();
6409        let records = records?;
6410        reconciled?;
6411        Ok(records)
6412    }
6413
6414    fn prepare_target_exact_source_selection(
6415        &mut self,
6416        sheet: &str,
6417        lease: StagedPackageLease,
6418        coordinates: Vec<(u32, u32)>,
6419        previous: &BTreeSet<(u32, u32)>,
6420        allow_partial_shared: bool,
6421        deadline: Option<std::time::Instant>,
6422        scratch: &mut u64,
6423    ) -> Result<Option<PreparedTargetSourcePackage>, ExcelError> {
6424        let package = self
6425            .staged_formulas
6426            .get(sheet)
6427            .and_then(|s| s.deferred_package.as_ref())
6428            .unwrap();
6429        if !package.source_geometry_complete
6430            || ((!package.families.is_empty() || !package.partitioned_families.is_empty())
6431                && !package.coordinates_cover_families)
6432            || package.reconciliation_replay.is_some()
6433        {
6434            return Ok(None);
6435        }
6436        let selected_points: BTreeSet<_> = coordinates
6437            .into_iter()
6438            .filter(|point| !package.suppressed.contains(point))
6439            .collect();
6440        let mut prepared = PreparedTargetSourcePackage::empty_selection(
6441            sheet,
6442            self.graph.sheet_id(sheet).unwrap(),
6443            lease,
6444        );
6445        if selected_points.is_empty() {
6446            return Ok(Some(prepared));
6447        }
6448        let mut routing = crate::engine::FormulaReplayDisposition::default();
6449        for partition in &package.partitioned_families {
6450            routing
6451                .register_partition(partition, false)
6452                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
6453        }
6454        routing.extend_suppressed_excel_coords(package.suppressed.iter().copied());
6455        routing.register_consumed_members(
6456            package.consumed_engine.as_ref(),
6457            package.consumed_members.iter().copied(),
6458        );
6459        let replay = Arc::clone(&package.replay);
6460        let source_report = package.accounting_report();
6461        prepared.source_report = source_report;
6462        prepared.disposition = routing.clone();
6463
6464        let replay_points: BTreeSet<_> = selected_points
6465            .iter()
6466            .copied()
6467            .filter(|&(row, col)| !prepared.direct_contains(row, col))
6468            .collect();
6469        let coordinates: Vec<_> = replay_points.iter().copied().collect();
6470        let records = self.replay_target_coordinates(&replay, &coordinates, deadline, scratch)?;
6471        prepared.spool_replays = u64::from(!coordinates.is_empty());
6472        let Some(mut replay_records) = records else {
6473            return Ok(None);
6474        };
6475        for record in &mut replay_records {
6476            if record.family.is_none() {
6477                record.partition_owner = routing
6478                    .ordinary_disposition(crate::engine::SourceCoord {
6479                        row: record.row.saturating_sub(1),
6480                        col: record.col.saturating_sub(1),
6481                    })
6482                    .1;
6483            }
6484        }
6485        replay_records.sort_by_key(|record| record.source_order);
6486        if replay_records
6487            .iter()
6488            .any(|record| !selected_points.contains(&(record.row, record.col)))
6489            || replay_records
6490                .windows(2)
6491                .any(|records| records[0].source_order == records[1].source_order)
6492        {
6493            return Err(ExcelError::new(ExcelErrorKind::Value)
6494                .with_message("invalid indexed exact source selection"));
6495        }
6496        let represented: BTreeSet<_> = replay_records.iter().map(|r| (r.row, r.col)).collect();
6497        if represented != replay_points {
6498            return Err(ExcelError::new(ExcelErrorKind::Value)
6499                .with_message("incomplete indexed exact source selection"));
6500        }
6501        // The last source record must agree with compressed ownership evidence.
6502        // Earlier overridden records retain ordering but demand no dependencies.
6503        let source = self
6504            .staged_formulas
6505            .get(sheet)
6506            .unwrap()
6507            .deferred_package
6508            .as_ref()
6509            .unwrap();
6510        let contains = |rect: crate::engine::SourceRect, coord: crate::engine::SourceCoord| {
6511            coord.row >= rect.start.row
6512                && coord.row <= rect.end.row
6513                && coord.col >= rect.start.col
6514                && coord.col <= rect.end.col
6515        };
6516        let mut checked = BTreeSet::new();
6517        for record in replay_records.iter().rev() {
6518            if !checked.insert((record.row, record.col)) {
6519                continue;
6520            }
6521            let coord = crate::engine::SourceCoord {
6522                row: record.row - 1,
6523                col: record.col - 1,
6524            };
6525            let agrees = |owner, shared: bool| {
6526                record.family == shared.then_some(owner)
6527                    && record.partition_owner.or(record.family) == Some(owner)
6528            };
6529            let mut valid = true;
6530            for family in &source.families {
6531                let owns = match &family.members {
6532                    crate::engine::SourceFamilyMembers::CompleteDomain(domain) => {
6533                        contains(domain.rect(), coord)
6534                    }
6535                    crate::engine::SourceFamilyMembers::ExplicitMembers(members) => {
6536                        members.as_slice().binary_search(&coord).is_ok()
6537                    }
6538                };
6539                if owns && !agrees(family.source_id, true) {
6540                    valid = false;
6541                }
6542            }
6543            for family in &source.partitioned_families {
6544                if family
6545                    .fragments
6546                    .iter()
6547                    .any(|fragment| contains(fragment.rect(), coord))
6548                    && !agrees(family.source_id, true)
6549                {
6550                    valid = false;
6551                }
6552                if let Some(member) = family
6553                    .legacy_members
6554                    .as_slice()
6555                    .iter()
6556                    .find(|member| member.coord == coord)
6557                    && !agrees(
6558                        family.source_id,
6559                        member.kind == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember,
6560                    )
6561                {
6562                    valid = false;
6563                }
6564            }
6565            if !valid {
6566                return Err(ExcelError::new(ExcelErrorKind::Value)
6567                    .with_message("indexed source ownership mismatch"));
6568            }
6569        }
6570        prepared.selected_points = Some(selected_points);
6571        prepared.replay_records = replay_records;
6572        Ok(Some(prepared))
6573    }
6574
6575    fn prepare_target_source_package(
6576        &mut self,
6577        sheet: &str,
6578        lease: StagedPackageLease,
6579        deadline: Option<std::time::Instant>,
6580    ) -> Result<PreparedTargetSourcePackage, ExcelError> {
6581        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
6582            ExcelError::new(ExcelErrorKind::Ref)
6583                .with_message(format!("deferred source sheet not found: {sheet}"))
6584        })?;
6585        let (
6586            source_report,
6587            families,
6588            partitions,
6589            replay,
6590            invalidated,
6591            suppressed,
6592            consumed_members,
6593            consumed_engine,
6594            reconciliation_replay,
6595        ) = {
6596            let package = self
6597                .staged_formulas
6598                .get(sheet)
6599                .and_then(|staged| staged.deferred_package.as_ref())
6600                .ok_or_else(|| {
6601                    ExcelError::new(ExcelErrorKind::Value)
6602                        .with_message("staged deferred source package is unavailable")
6603                })?;
6604            if package.sheet_name != sheet {
6605                return Err(ExcelError::new(ExcelErrorKind::Value)
6606                    .with_message("deferred formula package sheet mismatch"));
6607            }
6608            (
6609                package.accounting_report(),
6610                package.families.clone(),
6611                package.partitioned_families.clone(),
6612                Arc::clone(&package.replay),
6613                package.invalidated.clone(),
6614                package.suppressed.clone(),
6615                package.consumed_members.clone(),
6616                package.consumed_engine.clone(),
6617                package.reconciliation_replay.clone(),
6618            )
6619        };
6620
6621        let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
6622        for partition in &partitions {
6623            replay_disposition
6624                .register_partition(partition, false)
6625                .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
6626        }
6627        replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
6628        replay_disposition
6629            .register_consumed_members(consumed_engine.as_ref(), consumed_members.iter().copied());
6630        if !replay_disposition.consumed_engine_matches(&self.source_formula_token) {
6631            return Err(ExcelError::new(ExcelErrorKind::Value)
6632                .with_message("ResidualConsumedEngineMismatch"));
6633        }
6634        self.target_preparation_checkpoint(deadline, 1)?;
6635        let mut replay_records = if let Some(mut records) = reconciliation_replay {
6636            records.retain(|record| {
6637                let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
6638                else {
6639                    return true;
6640                };
6641                let coord = crate::engine::SourceCoord { row, col };
6642                let disposition = record.family.map_or_else(
6643                    || replay_disposition.ordinary_disposition(coord).0,
6644                    |family| replay_disposition.shared_disposition(family, coord),
6645                );
6646                !matches!(
6647                    disposition,
6648                    crate::engine::FormulaReplayCoordinateDisposition::Direct
6649                        | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
6650                )
6651            });
6652            records
6653        } else {
6654            replay
6655                .lock()
6656                .map_err(|_| {
6657                    ExcelError::new(ExcelErrorKind::Value)
6658                        .with_message("deferred formula spool lock poisoned")
6659                })?
6660                .replay_partitioned(&replay_disposition, &partitions)
6661                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
6662        };
6663        replay_records.sort_by_key(|record| record.source_order);
6664        for chunk in replay_records.chunks(256) {
6665            self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
6666        }
6667        if replay_records
6668            .windows(2)
6669            .any(|records| records[0].source_order == records[1].source_order)
6670        {
6671            return Err(ExcelError::new(ExcelErrorKind::Value)
6672                .with_message("duplicate deferred source-order proof"));
6673        }
6674
6675        let disposition = replay_disposition;
6676        let direct_families = 0usize;
6677        let direct_cells = 0u64;
6678        let direct_fragments = 0u64;
6679        let direct_complete_families = 0u64;
6680        let direct_complete_cells = 0u64;
6681        let direct_partition_families = 0u64;
6682        let direct_partition_cells = 0u64;
6683        let anchor_parses = 0u64;
6684        let anchor_asts = 0u64;
6685        let anchor_analyses = 0u64;
6686
6687        Ok(PreparedTargetSourcePackage {
6688            sheet: sheet.to_string(),
6689            sheet_id,
6690            lease,
6691            selected_points: None,
6692            complete_selections: Default::default(),
6693            deferred_shared: false,
6694            direct_domains: Vec::new(),
6695            source_report,
6696            replay_records,
6697            spool_replays: 1,
6698            disposition,
6699            legacy: Vec::new(),
6700            direct_families,
6701            direct_cells,
6702            direct_fragments,
6703            direct_complete_families,
6704            direct_complete_cells,
6705            direct_partition_families,
6706            direct_partition_cells,
6707            anchor_parses,
6708            anchor_asts,
6709            anchor_analyses,
6710        })
6711    }
6712
6713    fn fallback_planning_snapshot(
6714        &self,
6715        batch: &FormulaIngestBatch,
6716    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
6717        let mut requests = Vec::new();
6718        for formula in &batch.formulas {
6719            let ast = self
6720                .graph
6721                .data_store()
6722                .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
6723                .ok_or_else(|| {
6724                    ExcelError::new(ExcelErrorKind::Value)
6725                        .with_message("ordered fallback AST is unavailable")
6726                })?;
6727            Self::collect_planning_function_requests(&ast, &mut requests);
6728        }
6729        requests.sort();
6730        requests.dedup();
6731        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6732            &self.resolver,
6733            requests,
6734        )
6735        .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
6736    }
6737
6738    fn prepare_legacy_batch_fallback(
6739        &mut self,
6740        batch: FormulaIngestBatch,
6741        function_provider: &dyn crate::traits::FunctionProvider,
6742    ) -> Result<
6743        (
6744            crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
6745            u64,
6746        ),
6747        ExcelError,
6748    > {
6749        let formula_count = batch.formulas.len() as u64;
6750        let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6751        let mut planned = Vec::with_capacity(batch.formulas.len());
6752        for record in batch.formulas {
6753            let placement = CellRef::new(
6754                sheet_id,
6755                Coord::from_excel(record.row, record.col, true, true),
6756            );
6757            let ingested = self
6758                .graph
6759                .ingest_pipeline(function_provider)
6760                .enable_function_semantics()
6761                .ingest_formula(
6762                    FormulaAstInput::RawArena(record.ast_id),
6763                    placement,
6764                    record.formula_text,
6765                )
6766                .map_err(|error| {
6767                    ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
6768                })?;
6769            planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
6770        }
6771        let plan = self
6772            .graph
6773            .prepare_legacy_graph_plan(sheet_id, planned)
6774            .map_err(|error| {
6775                ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
6776            })?;
6777        Ok((plan, formula_count))
6778    }
6779
6780    fn publish_compressed_partial_report(
6781        &mut self,
6782        report: &FormulaIngestReport,
6783        direct_report: &FormulaIngestReport,
6784    ) {
6785        if direct_report.source_family_promoted == 0
6786            && direct_report.graph_formula_cells_materialized == 0
6787        {
6788            return;
6789        }
6790        let mut published = report.clone();
6791        published.accumulate(direct_report);
6792        self.record_formula_ingest_report(published);
6793    }
6794
6795    fn finish_compressed_formula_sources(
6796        &mut self,
6797        batches: Vec<(
6798            FormulaIngestBatch,
6799            crate::engine::FormulaCompressedSourceReport,
6800            crate::engine::FormulaCompressedPreparation,
6801        )>,
6802    ) -> Result<FormulaIngestReport, ExcelError> {
6803        self.observe_function_semantic_epoch()?;
6804        if batches.iter().any(|(_, _, preparation)| {
6805            !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
6806        }) {
6807            return Err(ExcelError::new(ExcelErrorKind::Value)
6808                .with_message("compressed source preparation belongs to another engine"));
6809        }
6810        if batches.iter().any(|(fallback, _, preparation)| {
6811            preparation.sheet_name.as_ref() != fallback.sheet_name
6812        }) {
6813            return Err(ExcelError::new(ExcelErrorKind::Value)
6814                .with_message("compressed source preparation sheet mismatch"));
6815        }
6816        let initial_guard = crate::function_registry::semantic_epoch_read_guard();
6817        let initial_provider_revision = self.resolver.planning_semantic_revision();
6818        let mut fallback_batches = Vec::with_capacity(batches.len());
6819        let mut stale_fallback_batches = Vec::new();
6820        let mut pending_preparations = Vec::new();
6821        for (mut fallback, mut source, mut preparation) in batches {
6822            for formula in fallback.formulas.drain(..) {
6823                let source_order = formula.source_order.ok_or_else(|| {
6824                    ExcelError::new(ExcelErrorKind::Value).with_message(
6825                        "compressed source supplied formulas without source-order proof",
6826                    )
6827                })?;
6828                let text = formula.formula_text.ok_or_else(|| {
6829                    ExcelError::new(ExcelErrorKind::Value).with_message(
6830                        "ordered compressed fallback formula has no exact source text",
6831                    )
6832                })?;
6833                preparation
6834                    .eager_replay
6835                    .push(crate::engine::DeferredReplayFormula {
6836                        source_order,
6837                        row: formula.row,
6838                        col: formula.col,
6839                        text: text.to_string(),
6840                        family: formula.source_family,
6841                        partition_owner: formula.partition_owner,
6842                    });
6843            }
6844            preparation
6845                .eager_replay
6846                .sort_by_key(|record| record.source_order);
6847            source.source_spool_replays = source
6848                .source_spool_replays
6849                .saturating_add(preparation.preparation_spool_replays);
6850            let stale_reason = preparation
6851                .function_semantics_used
6852                .then(|| {
6853                    if preparation.function_provider_revision != initial_provider_revision {
6854                        Some("FunctionProviderRevisionChanged")
6855                    } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
6856                    {
6857                        Some("FunctionSemanticEpochChanged")
6858                    } else {
6859                        None
6860                    }
6861                })
6862                .flatten();
6863            let stale_semantics = stale_reason.is_some();
6864            for reason in preparation.rejected.values() {
6865                *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
6866            }
6867            // No family is ever placed directly: every family replays.
6868            source.replay_families = source.families_seen;
6869            source.replay_cells = source.family_cells_seen;
6870            let compressed = crate::engine::FormulaCompressedSourceBatch::new(
6871                fallback.sheet_name.clone(),
6872                source,
6873            );
6874            if stale_semantics {
6875                stale_fallback_batches.push((fallback, compressed));
6876            } else {
6877                fallback_batches.push((fallback, compressed));
6878            }
6879            pending_preparations.push((preparation, stale_semantics));
6880        }
6881        drop(initial_guard);
6882
6883        // Build known fallback graphs first. Stale batches are forced through legacy ingest.
6884        let configured_mode = self.config.formula_plane_mode;
6885        self.config.formula_plane_mode = FormulaPlaneMode::Off;
6886        let stale_result =
6887            self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
6888        self.config.formula_plane_mode = configured_mode;
6889        let mut report = stale_result?;
6890        report.mode = configured_mode;
6891
6892        self.config.formula_plane_mode = FormulaPlaneMode::Off;
6893        let fallback_result =
6894            self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
6895        self.config.formula_plane_mode = configured_mode;
6896        match fallback_result {
6897            Ok(fallback_report) => report.accumulate(&fallback_report),
6898            Err(error) => {
6899                self.record_formula_ingest_report(report);
6900                return Err(error);
6901            }
6902        }
6903
6904        let mut direct_report =
6905            FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
6906        for (preparation, _) in &pending_preparations {
6907            direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
6908                preparation.clean_rejected_anchor_counts[0]
6909                    .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
6910            );
6911            direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
6912                preparation.clean_rejected_anchor_counts[1]
6913                    .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
6914            );
6915            direct_report.source_anchor_analyses =
6916                direct_report.source_anchor_analyses.saturating_add(
6917                    preparation.clean_rejected_anchor_counts[2]
6918                        .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
6919                );
6920        }
6921        loop {
6922            #[cfg(any(test, feature = "test-support"))]
6923            if let Some(hook) = self.before_prepared_span_commit_hook.take() {
6924                hook();
6925            }
6926            let commit_guard = crate::function_registry::semantic_epoch_read_guard();
6927            let commit_provider_revision = self.resolver.planning_semantic_revision();
6928            let mut newly_stale = Vec::new();
6929            let mut current = Vec::new();
6930            for pending in pending_preparations.drain(..) {
6931                let stale_reason = pending
6932                    .0
6933                    .function_semantics_used
6934                    .then(|| {
6935                        if pending.0.function_provider_revision != commit_provider_revision {
6936                            Some("FunctionProviderRevisionChanged")
6937                        } else if self
6938                            .prepared_function_semantics_changed(&pending.0, &commit_guard)
6939                        {
6940                            Some("FunctionSemanticEpochChanged")
6941                        } else {
6942                            None
6943                        }
6944                    })
6945                    .flatten();
6946                if let Some(reason) = stale_reason {
6947                    newly_stale.push((pending, reason));
6948                } else {
6949                    current.push(pending);
6950                }
6951            }
6952            if !newly_stale.is_empty() {
6953                drop(commit_guard);
6954                for ((mut preparation, was_initially_stale), reason) in newly_stale {
6955                    preparation
6956                        .eager_replay
6957                        .sort_by_key(|record| record.source_order);
6958                    direct_report.source_spool_replays =
6959                        direct_report.source_spool_replays.saturating_add(1);
6960                    if !was_initially_stale {
6961                        direct_report
6962                            .fallback_reasons
6963                            .entry(reason.to_string())
6964                            .or_default();
6965                    }
6966                    preparation.function_semantics_used = false;
6967                    if !preparation.eager_replay.is_empty() {
6968                        current.push((preparation, false));
6969                    }
6970                }
6971                pending_preparations = current;
6972                continue;
6973            }
6974            drop(commit_guard);
6975
6976            // Replay fallback families in source order.
6977            enum SourceProposal {
6978                KnownFallback {
6979                    source_order: crate::engine::SourceFormulaOrder,
6980                    records: Vec<crate::engine::DeferredReplayFormula>,
6981                },
6982            }
6983
6984            impl SourceProposal {
6985                fn source_order(&self) -> crate::engine::SourceFormulaOrder {
6986                    match self {
6987                        Self::KnownFallback { source_order, .. } => *source_order,
6988                    }
6989                }
6990            }
6991
6992            for (mut preparation, _) in current {
6993                let mut proposals = Vec::with_capacity(preparation.eager_replay.len());
6994                let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
6995                for record in preparation.eager_replay.drain(..) {
6996                    if let Some(owner) = record.partition_owner.or(record.family) {
6997                        family_fallbacks.entry(owner).or_default().push(record);
6998                    } else {
6999                        proposals.push(SourceProposal::KnownFallback {
7000                            source_order: record.source_order,
7001                            records: vec![record],
7002                        });
7003                    }
7004                }
7005                for (_, mut records) in family_fallbacks {
7006                    records.sort_by_key(|record| record.source_order);
7007                    if records
7008                        .windows(2)
7009                        .any(|window| window[0].source_order == window[1].source_order)
7010                    {
7011                        self.publish_compressed_partial_report(&report, &direct_report);
7012                        return Err(ExcelError::new(ExcelErrorKind::Value)
7013                            .with_message("duplicate exact-replay source-order proof"));
7014                    }
7015                    let Some(source_order) = records.first().map(|record| record.source_order)
7016                    else {
7017                        self.publish_compressed_partial_report(&report, &direct_report);
7018                        return Err(ExcelError::new(ExcelErrorKind::Value)
7019                            .with_message("empty exact-replay fallback family"));
7020                    };
7021                    proposals.push(SourceProposal::KnownFallback {
7022                        source_order,
7023                        records,
7024                    });
7025                }
7026                proposals.sort_by_key(SourceProposal::source_order);
7027                if proposals
7028                    .windows(2)
7029                    .any(|window| window[0].source_order() == window[1].source_order())
7030                {
7031                    self.publish_compressed_partial_report(&report, &direct_report);
7032                    return Err(ExcelError::new(ExcelErrorKind::Value)
7033                        .with_message("ambiguous compressed source-order proof"));
7034                }
7035
7036                for proposal in proposals {
7037                    match proposal {
7038                        SourceProposal::KnownFallback { records, .. } => {
7039                            let batch = match self.formula_batch_from_exact_replay(
7040                                preparation.sheet_name.as_ref(),
7041                                records,
7042                            ) {
7043                                Ok(batch) => batch,
7044                                Err(error) => {
7045                                    self.publish_compressed_partial_report(&report, &direct_report);
7046                                    return Err(error);
7047                                }
7048                            };
7049                            if batch.is_empty() {
7050                                continue;
7051                            }
7052                            let snapshot = match self.fallback_planning_snapshot(&batch) {
7053                                Ok(snapshot) => snapshot,
7054                                Err(error) => {
7055                                    self.publish_compressed_partial_report(&report, &direct_report);
7056                                    return Err(error);
7057                                }
7058                            };
7059                            let commit_guard =
7060                                crate::function_registry::semantic_epoch_read_guard();
7061                            let provider_revision_initial =
7062                                self.resolver.planning_semantic_revision();
7063                            if (commit_guard.epoch() != snapshot.epoch()
7064                                && snapshot.semantic_changes_affect_requests_since_guarded(
7065                                    &commit_guard,
7066                                    snapshot.epoch(),
7067                                ))
7068                                || snapshot.provider_revision().is_some_and(|revision| {
7069                                    Some(revision) != provider_revision_initial
7070                                })
7071                            {
7072                                drop(commit_guard);
7073                                self.publish_compressed_partial_report(&report, &direct_report);
7074                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7075                                    "ordered fallback planning snapshot became stale",
7076                                ));
7077                            }
7078                            let (plan, formula_count) = match self
7079                                .prepare_legacy_batch_fallback(batch, &snapshot)
7080                            {
7081                                Ok(plan) => plan,
7082                                Err(error) => {
7083                                    drop(commit_guard);
7084                                    self.publish_compressed_partial_report(&report, &direct_report);
7085                                    return Err(error);
7086                                }
7087                            };
7088                            let provider_revision_after =
7089                                self.resolver.planning_semantic_revision();
7090                            if provider_revision_after != provider_revision_initial {
7091                                drop(commit_guard);
7092                                self.publish_compressed_partial_report(&report, &direct_report);
7093                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7094                                    "function provider changed while preparing ordered fallback",
7095                                ));
7096                            }
7097                            let graph_vertices = plan.new_vertex_count();
7098                            let Some(graph_edges) = plan.planned_edge_count() else {
7099                                drop(commit_guard);
7100                                self.publish_compressed_partial_report(&report, &direct_report);
7101                                return Err(ExcelError::new(ExcelErrorKind::Value)
7102                                    .with_message("prepared ordered fallback size overflow"));
7103                            };
7104                            if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
7105                            {
7106                                drop(commit_guard);
7107                                self.publish_compressed_partial_report(&report, &direct_report);
7108                                return Err(error);
7109                            }
7110                            if let Err(error) =
7111                                self.graph.validate_prepared_legacy_graph_plan(&plan)
7112                            {
7113                                drop(commit_guard);
7114                                self.publish_compressed_partial_report(&report, &direct_report);
7115                                return Err(ExcelError::new(ExcelErrorKind::Value)
7116                                    .with_message(error.to_string()));
7117                            }
7118                            #[cfg(test)]
7119                            if let Some(hook) = self
7120                                .before_legacy_fallback_final_provider_sample_hook
7121                                .take()
7122                            {
7123                                hook();
7124                            }
7125                            let provider_revision_final =
7126                                self.resolver.planning_semantic_revision();
7127                            if provider_revision_final != provider_revision_initial {
7128                                drop(commit_guard);
7129                                self.publish_compressed_partial_report(&report, &direct_report);
7130                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7131                                    "function provider changed after ordered fallback validation",
7132                                ));
7133                            }
7134                            let graph_formulas =
7135                                self.graph.apply_prevalidated_legacy_graph_plan(plan);
7136                            direct_report.formula_cells_seen = direct_report
7137                                .formula_cells_seen
7138                                .saturating_add(formula_count);
7139                            direct_report.graph_formula_cells_materialized = direct_report
7140                                .graph_formula_cells_materialized
7141                                .saturating_add(graph_formulas as u64);
7142                            direct_report.graph_vertices_created = direct_report
7143                                .graph_vertices_created
7144                                .saturating_add(graph_vertices as u64);
7145                            direct_report.graph_edges_created = direct_report
7146                                .graph_edges_created
7147                                .saturating_add(graph_edges as u64);
7148                        }
7149                    }
7150                    #[cfg(test)]
7151                    if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
7152                        hook();
7153                    }
7154                }
7155            }
7156            break;
7157        }
7158        report.accumulate(&direct_report);
7159        self.record_formula_ingest_report(report.clone());
7160        Ok(report)
7161    }
7162    /// Ingest replayed per-cell formulas while preserving compressed source counters.
7163    pub(crate) fn ingest_compressed_formula_source_batches(
7164        &mut self,
7165        batches: Vec<(
7166            FormulaIngestBatch,
7167            crate::engine::FormulaCompressedSourceBatch,
7168        )>,
7169    ) -> Result<FormulaIngestReport, ExcelError> {
7170        self.ingest_compressed_formula_source_batches_inner(batches, true)
7171    }
7172
7173    fn ingest_compressed_formula_source_batches_inner(
7174        &mut self,
7175        batches: Vec<(
7176            FormulaIngestBatch,
7177            crate::engine::FormulaCompressedSourceBatch,
7178        )>,
7179        publish_report: bool,
7180    ) -> Result<FormulaIngestReport, ExcelError> {
7181        let mut source_counts = [0_u64; 11];
7182        let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
7183        let mut formula_batches = Vec::with_capacity(batches.len());
7184        let mut compressed_families = Vec::new();
7185        let mut partitioned_families = Vec::new();
7186        for (batch, compressed_batch) in batches {
7187            let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
7188            if sheet_name.as_ref() != batch.sheet_name {
7189                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7190                    "compressed formula source sheet does not match its replay batch",
7191                ));
7192            }
7193            compressed_families.push((batch.sheet_name.clone(), families));
7194            partitioned_families.push((batch.sheet_name.clone(), partitions));
7195            source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
7196            source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
7197            source_counts[2] =
7198                source_counts[2].saturating_add(compressed.source_shared_anchor_events);
7199            source_counts[3] =
7200                source_counts[3].saturating_add(compressed.source_shared_descendant_events);
7201            source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
7202            source_counts[5] =
7203                source_counts[5].saturating_add(compressed.source_formula_records_spooled);
7204            source_counts[6] =
7205                source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
7206            source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
7207            source_counts[8] =
7208                source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
7209            source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
7210            source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
7211            source_report.families_seen = source_report
7212                .families_seen
7213                .saturating_add(compressed.families_seen);
7214            source_report.family_cells_seen = source_report
7215                .family_cells_seen
7216                .saturating_add(compressed.family_cells_seen);
7217            source_report.source_clean_families = source_report
7218                .source_clean_families
7219                .saturating_add(compressed.source_clean_families);
7220            source_report.source_clean_cells = source_report
7221                .source_clean_cells
7222                .saturating_add(compressed.source_clean_cells);
7223            source_report.source_fragmentable_families = source_report
7224                .source_fragmentable_families
7225                .saturating_add(compressed.source_fragmentable_families);
7226            source_report.source_fragmentable_cells = source_report
7227                .source_fragmentable_cells
7228                .saturating_add(compressed.source_fragmentable_cells);
7229            source_report.source_fragment_count = source_report
7230                .source_fragment_count
7231                .saturating_add(compressed.source_fragment_count);
7232            source_report.source_isolated_fallback_cells = source_report
7233                .source_isolated_fallback_cells
7234                .saturating_add(compressed.source_isolated_fallback_cells);
7235            source_report.source_hole_exclusions = source_report
7236                .source_hole_exclusions
7237                .saturating_add(compressed.source_hole_exclusions);
7238            source_report.source_ordinary_exclusions = source_report
7239                .source_ordinary_exclusions
7240                .saturating_add(compressed.source_ordinary_exclusions);
7241            source_report.source_partition_failures = source_report
7242                .source_partition_failures
7243                .saturating_add(compressed.source_partition_failures);
7244            source_report.replay_families = source_report
7245                .replay_families
7246                .saturating_add(compressed.replay_families);
7247            source_report.replay_cells = source_report
7248                .replay_cells
7249                .saturating_add(compressed.replay_cells);
7250            source_report.forward_descendants = source_report
7251                .forward_descendants
7252                .saturating_add(compressed.forward_descendants);
7253            source_report.evidence_limit_fallbacks = source_report
7254                .evidence_limit_fallbacks
7255                .saturating_add(compressed.evidence_limit_fallbacks);
7256            source_report.evidence_peak_bytes = source_report
7257                .evidence_peak_bytes
7258                .max(compressed.evidence_peak_bytes);
7259            for (reason, count) in compressed.fallback_reasons {
7260                *source_report.fallback_reasons.entry(reason).or_default() += count;
7261            }
7262            formula_batches.push(batch);
7263        }
7264        self.ingest_formula_batches_inner(
7265            formula_batches,
7266            source_counts,
7267            Some(source_report),
7268            compressed_families,
7269            partitioned_families,
7270            publish_report,
7271        )
7272    }
7273
7274    pub fn ingest_formula_batches(
7275        &mut self,
7276        batches: Vec<FormulaIngestBatch>,
7277    ) -> Result<FormulaIngestReport, ExcelError> {
7278        let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
7279        let report = self.ingest_formula_batches_inner(
7280            batches,
7281            [0; 11],
7282            None,
7283            Vec::new(),
7284            Vec::new(),
7285            true,
7286        )?;
7287        if has_formulas {
7288            self.mark_topology_edited();
7289        }
7290        Ok(report)
7291    }
7292
7293    fn ingest_formula_batches_unpublished(
7294        &mut self,
7295        batches: Vec<FormulaIngestBatch>,
7296    ) -> Result<FormulaIngestReport, ExcelError> {
7297        self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
7298    }
7299
7300    fn ingest_formula_batches_inner(
7301        &mut self,
7302        batches: Vec<FormulaIngestBatch>,
7303        source_counts: [u64; 11],
7304        source_report: Option<crate::engine::FormulaCompressedSourceReport>,
7305        compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
7306        partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
7307        publish_report: bool,
7308    ) -> Result<FormulaIngestReport, ExcelError> {
7309        self.observe_function_semantic_epoch()?;
7310        let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7311        #[cfg(feature = "tracing")]
7312        let arena_nodes_before = self.graph.data_store().memory_usage().total_ast_nodes;
7313        #[cfg(feature = "tracing")]
7314        let route = if !partitioned_families.is_empty() {
7315            "partitioned"
7316        } else if !compressed_families.is_empty() {
7317            "compressed_source"
7318        } else if source_report.is_some() {
7319            "replay"
7320        } else {
7321            "ordinary"
7322        };
7323        let _ingest_span = crate::engine::trace::fz_span!(
7324            tracing::Level::INFO,
7325            "ingest",
7326            "ingest.batch",
7327            mode = ?FormulaPlaneMode::Off,
7328            route,
7329            formula_cells = formula_cells_seen
7330        );
7331        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
7332        let materialize_batches = batches;
7333        report.formula_cells_seen = formula_cells_seen;
7334        report.source_formula_events = source_counts[0];
7335        report.source_ordinary_events = source_counts[1];
7336        report.source_shared_anchor_events = source_counts[2];
7337        report.source_shared_descendant_events = source_counts[3];
7338        report.source_unknown_events = source_counts[4];
7339        report.source_formula_records_spooled = source_counts[5];
7340        report.source_spool_encoded_bytes = source_counts[6];
7341        report.source_spool_peak_memory_bytes = source_counts[7];
7342        report.source_spool_spilled_bytes = source_counts[8];
7343        report.source_spool_spill_files = source_counts[9];
7344        report.source_spool_replays = source_counts[10];
7345        if let Some(source) = source_report {
7346            report.source_families_seen = source.families_seen;
7347            report.source_family_cells_seen = source.family_cells_seen;
7348            report.source_family_shadow_eligible = source.source_clean_families;
7349            report.source_family_shadow_eligible_cells = source.source_clean_cells;
7350            report.source_partitioned_families_seen = source.source_fragmentable_families;
7351            report.source_partition_holes = source.source_hole_exclusions;
7352            report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
7353            report.source_partition_failures = source.source_partition_failures;
7354            report.source_partition_surviving_cells = source.source_fragmentable_cells;
7355            report.source_family_fallback = report
7356                .source_family_fallback
7357                .saturating_add(source.replay_families);
7358            report.source_family_fallback_cells = report
7359                .source_family_fallback_cells
7360                .saturating_add(source.replay_cells);
7361            report.source_forward_descendants = source.forward_descendants;
7362            report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
7363            report.source_evidence_peak_bytes = source.evidence_peak_bytes;
7364            for (reason, count) in source.fallback_reasons {
7365                let total = report.fallback_reasons.entry(reason).or_default();
7366                *total = total.saturating_add(count);
7367            }
7368        }
7369
7370        // A first load without graph admission plans in the builder, one
7371        // chunk at a time (a load error fails the load, so the builder's
7372        // partial application on a planning error is unobservable).
7373        if self.graph.first_load_assume_new()
7374            && !self.graph_admission_enabled()
7375            && !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
7376        {
7377            let mut builder =
7378                crate::engine::ingest_builder::BulkIngestBuilder::new(&mut self.graph);
7379            for batch in materialize_batches {
7380                if batch.is_empty() {
7381                    continue;
7382                }
7383                let sheet_id = builder
7384                    .add_sheet_checked(&batch.sheet_name)
7385                    .ok_or_else(|| {
7386                        ExcelError::new(ExcelErrorKind::Ref)
7387                            .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7388                    })?;
7389                builder.add_formula_refs(
7390                    sheet_id,
7391                    batch.formulas.into_iter().map(|record| {
7392                        (
7393                            record.row,
7394                            record.col,
7395                            crate::engine::graph::FormulaRef::of_ingested(
7396                                record.ast_id,
7397                                record.member_anchor,
7398                            ),
7399                        )
7400                    }),
7401                );
7402            }
7403            let summary = builder.finish_with_provider(&self.resolver)?;
7404            report.graph_formula_cells_materialized = summary.formulas as u64;
7405            report.graph_vertices_created = summary.vertices as u64;
7406            report.graph_edges_created = summary.edges as u64;
7407        } else if !materialize_batches.iter().all(FormulaIngestBatch::is_empty) {
7408            let mut prepared_by_sheet: BTreeMap<String, Vec<_>> = BTreeMap::new();
7409            for batch in materialize_batches {
7410                if batch.is_empty() {
7411                    continue;
7412                }
7413                let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
7414                    ExcelError::new(ExcelErrorKind::Ref)
7415                        .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7416                })?;
7417                let mut pipeline = self.ingest_pipeline();
7418                // Plan each record and keep only what the graph needs (the
7419                // pipeline's per-formula facts are dropped at once).
7420                let prepared = prepared_by_sheet.entry(batch.sheet_name).or_default();
7421                prepared.reserve(batch.formulas.len());
7422                for record in batch.formulas {
7423                    let placement = CellRef::new(
7424                        sheet_id,
7425                        Coord::from_excel(record.row, record.col, true, true),
7426                    );
7427                    let input = match record.member_anchor {
7428                        Some(anchor) => FormulaAstInput::Member {
7429                            template: record.ast_id,
7430                            anchor,
7431                        },
7432                        None => FormulaAstInput::RawArena(record.ast_id),
7433                    };
7434                    let formula = pipeline.ingest_formula(input, placement, None)?;
7435                    prepared.push((
7436                        record.row,
7437                        record.col,
7438                        crate::engine::graph::FormulaRef::of_ingested(
7439                            formula.ast_id,
7440                            formula.member_anchor,
7441                        ),
7442                        formula.dep_plan,
7443                    ));
7444                }
7445            }
7446            let admission_preflighted = self.graph_admission_enabled();
7447            if admission_preflighted {
7448                let mut preview = Vec::new();
7449                for (sheet_name, formulas) in &prepared_by_sheet {
7450                    let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
7451                        ExcelError::new(ExcelErrorKind::Ref)
7452                            .with_message(format!("unknown ingest sheet: {sheet_name}"))
7453                    })?;
7454                    preview.extend(
7455                        formulas
7456                            .iter()
7457                            .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
7458                    );
7459                }
7460                let admission = self.graph.preview_formula_mutations(&preview)?;
7461                self.preflight_graph_admission(admission)?;
7462            }
7463
7464            let mut builder = self.begin_bulk_ingest();
7465            if admission_preflighted {
7466                builder.mark_admission_preflighted();
7467            }
7468            for (sheet_name, formulas) in prepared_by_sheet {
7469                if formulas.is_empty() {
7470                    continue;
7471                }
7472                let sheet_id = builder.add_sheet(&sheet_name);
7473                builder.add_formula_plans(sheet_id, formulas);
7474            }
7475            let summary = builder.finish()?;
7476            report.graph_formula_cells_materialized = summary.formulas as u64;
7477            report.graph_vertices_created = summary.vertices as u64;
7478            report.graph_edges_created = summary.edges as u64;
7479        }
7480
7481        crate::engine::trace::fz_event!(
7482            tracing::Level::INFO,
7483            "ingest",
7484            "ingest.summary",
7485            candidate_cells = report.shadow_candidate_cells,
7486            accepted_span_cells = report.shadow_accepted_span_cells,
7487            fallback_cells = report.shadow_fallback_cells,
7488            spans_created = report.shadow_spans_created,
7489            templates_interned = report.shadow_templates_interned,
7490            graph_vertices_created = report.graph_vertices_created,
7491            graph_edges_created = report.graph_edges_created,
7492            arena_nodes_delta = self
7493                .graph
7494                .data_store()
7495                .memory_usage()
7496                .total_ast_nodes
7497                .saturating_sub(arena_nodes_before)
7498        );
7499        if publish_report {
7500            self.record_formula_ingest_report(report.clone());
7501        }
7502        Ok(report)
7503    }
7504
7505    fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
7506        let mut unique = Vec::new();
7507        for diagnostic in self.formula_parse_diagnostics.drain(start..) {
7508            let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
7509                prior.sheet == diagnostic.sheet
7510                    && prior.row == diagnostic.row
7511                    && prior.col == diagnostic.col
7512                    && prior.formula == diagnostic.formula
7513                    && prior.policy == diagnostic.policy
7514            });
7515            if !duplicate {
7516                unique.push(diagnostic);
7517            }
7518        }
7519        self.formula_parse_diagnostics.extend(unique);
7520    }
7521
7522    pub fn handle_formula_parse_error(
7523        &mut self,
7524        sheet: &str,
7525        row: u32,
7526        col: u32,
7527        formula: &str,
7528        message: String,
7529    ) -> Result<Option<ASTNode>, ExcelError> {
7530        let policy = self.config.formula_parse_policy;
7531
7532        if policy == FormulaParsePolicy::Strict {
7533            let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
7534            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7535                "Formula parse error at {sheet}!{col_a1}{row}: {message}"
7536            )));
7537        }
7538
7539        self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
7540            sheet: sheet.to_string(),
7541            row,
7542            col,
7543            formula: formula.to_string(),
7544            message: message.clone(),
7545            policy,
7546        });
7547
7548        match policy {
7549            FormulaParsePolicy::Strict => unreachable!(),
7550            FormulaParsePolicy::KeepCachedValue => Ok(None),
7551            FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
7552                ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
7553                None,
7554            ))),
7555            FormulaParsePolicy::CoerceToError => {
7556                let err = ExcelError::new(ExcelErrorKind::Error)
7557                    .with_message(format!("Malformed formula: {message}"));
7558                Ok(Some(ASTNode::new(
7559                    ASTNodeType::Literal(LiteralValue::Error(err)),
7560                    None,
7561                )))
7562            }
7563        }
7564    }
7565
7566    #[cfg(test)]
7567    fn target_preparation_fault(
7568        &mut self,
7569        seam: crate::engine::target_preparation::TargetPreparationFault,
7570    ) -> Result<(), ExcelError> {
7571        if self.target_preparation_fault_for_test == Some(seam) {
7572            self.target_preparation_fault_for_test = None;
7573            Err(ExcelError::new(ExcelErrorKind::Value)
7574                .with_message(format!("injected target preparation fault: {seam:?}")))
7575        } else {
7576            Ok(())
7577        }
7578    }
7579
7580    fn preparation_stale(
7581        reason: formualizer_common::PreparationStaleReason,
7582        message: impl Into<String>,
7583    ) -> ExcelError {
7584        ExcelError::new(ExcelErrorKind::Value)
7585            .with_message(message)
7586            .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
7587    }
7588
7589    fn preparation_revision_stale_reason(
7590        assumptions: &crate::engine::PreparationRevision,
7591        current: &crate::engine::PreparationRevision,
7592        planning_requests: &BTreeSet<(String, String, usize)>,
7593        staged_leases_match: bool,
7594    ) -> Option<formualizer_common::PreparationStaleReason> {
7595        if assumptions.graph != current.graph {
7596            Some(formualizer_common::PreparationStaleReason::Graph)
7597        } else if assumptions.staged != current.staged || !staged_leases_match {
7598            Some(formualizer_common::PreparationStaleReason::Staged)
7599        } else if assumptions.symbols != current.symbols {
7600            Some(formualizer_common::PreparationStaleReason::Symbols)
7601        } else if assumptions.provider != current.provider {
7602            Some(formualizer_common::PreparationStaleReason::Provider)
7603        } else if assumptions.semantic != current.semantic
7604            && crate::function_registry::semantic_changes_affect_requests_since(
7605                assumptions.semantic,
7606                planning_requests.iter().cloned(),
7607            )
7608        {
7609            Some(formualizer_common::PreparationStaleReason::Semantic)
7610        } else {
7611            None
7612        }
7613    }
7614
7615    fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
7616        crate::engine::PreparationRevision {
7617            graph: self.graph.topology_revision(),
7618            authority: 0,
7619            authority_indexes: 0,
7620            authority_indexed_plane: 0,
7621            staged: self.staged_formula_index.revision(),
7622            symbols: self.graph.symbol_revision(),
7623            semantic: crate::function_registry::semantic_epoch(),
7624            provider: self.resolver.planning_semantic_revision(),
7625        }
7626    }
7627
7628    fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
7629        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
7630        let provider = self.resolver.planning_semantic_revision();
7631        let semantic = registry_guard.epoch();
7632        PlanningRevisionSnapshot {
7633            engine_topology_epoch: self.topology_epoch,
7634            graph_topology_revision: self.graph.topology_revision(),
7635            staged: self.staged_formula_index.revision(),
7636            symbols: self.graph.symbol_revision(),
7637            semantic,
7638            provider,
7639            deterministic_mode: self.config.deterministic_mode.clone(),
7640            budgets: self.evaluation_resource_budgets.clone(),
7641        }
7642    }
7643
7644    fn recalc_plan_key(&self) -> RecalcPlanKey {
7645        RecalcPlanKey {
7646            engine_token: Arc::clone(&self.recalc_plan_token),
7647            revisions: self.planning_revision_snapshot(),
7648        }
7649    }
7650
7651    fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
7652        ExcelError::new(ExcelErrorKind::Value)
7653            .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
7654            .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
7655    }
7656
7657    fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
7658        use formualizer_common::PlanStaleReason;
7659
7660        if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
7661            return Err(Self::plan_stale(PlanStaleReason::Engine));
7662        }
7663
7664        let current = self.planning_revision_snapshot();
7665        let expected = &key.revisions;
7666        let stale = if expected.provider != current.provider {
7667            Some(PlanStaleReason::Provider)
7668        } else if expected.semantic != current.semantic {
7669            Some(PlanStaleReason::Semantic)
7670        } else if expected.budgets != current.budgets
7671            || expected.deterministic_mode != current.deterministic_mode
7672        {
7673            Some(PlanStaleReason::Budget)
7674        } else if expected.staged != current.staged {
7675            Some(PlanStaleReason::Staged)
7676        } else if expected.symbols != current.symbols {
7677            Some(PlanStaleReason::Symbols)
7678        } else if expected.graph_topology_revision != current.graph_topology_revision
7679            || expected.engine_topology_epoch != current.engine_topology_epoch
7680        {
7681            Some(PlanStaleReason::Graph)
7682        } else {
7683            None
7684        };
7685        stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
7686    }
7687
7688    fn target_preparation_checkpoint(
7689        &mut self,
7690        deadline: Option<std::time::Instant>,
7691        work: u64,
7692    ) -> Result<(), ExcelError> {
7693        if self
7694            .active_cancel_flag
7695            .as_ref()
7696            .is_some_and(|cancel| cancel.is_cancelled())
7697        {
7698            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
7699                .with_message("target graph preparation cancelled"));
7700        }
7701        if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
7702            return Err(crate::engine::ResourceLedgerError::Exhausted(
7703                formualizer_common::ResourceExhaustionDetail {
7704                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
7705                    limit: 0,
7706                    observed: 1,
7707                    request_id: self
7708                        .active_evaluation_resource_request
7709                        .as_ref()
7710                        .map(|stats| stats.request_id),
7711                },
7712            )
7713            .into_excel_error());
7714        }
7715        self.charge_bounded_work(work)
7716    }
7717
7718    fn opaque_reason_in_ast(
7719        &self,
7720        ast: &ASTNode,
7721        provider: &dyn crate::traits::FunctionProvider,
7722    ) -> Option<crate::engine::OpaqueReason> {
7723        match &ast.node_type {
7724            ASTNodeType::Function { name, args } => {
7725                let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
7726                if canonical == "INDIRECT" {
7727                    return Some(crate::engine::OpaqueReason::RuntimeTextReference);
7728                }
7729                let Some(function) = provider.get_function_for_planning("", name) else {
7730                    return Some(crate::engine::OpaqueReason::UnknownFunction);
7731                };
7732                let caps = function.caps();
7733                if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
7734                    || caps.contains(FnCaps::RETURNS_REFERENCE)
7735                {
7736                    return Some(crate::engine::OpaqueReason::DynamicReference);
7737                }
7738                args.iter()
7739                    .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
7740            }
7741            ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
7742            ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
7743            ASTNodeType::BinaryOp { left, right, .. } => self
7744                .opaque_reason_in_ast(left, provider)
7745                .or_else(|| self.opaque_reason_in_ast(right, provider)),
7746            ASTNodeType::Array(rows) => rows
7747                .iter()
7748                .flat_map(|row| row.iter())
7749                .find_map(|item| self.opaque_reason_in_ast(item, provider)),
7750            ASTNodeType::Reference {
7751                reference:
7752                    ReferenceType::Cell {
7753                        sheet: Some(sheet), ..
7754                    }
7755                    | ReferenceType::Range {
7756                        sheet: Some(sheet), ..
7757                    },
7758                ..
7759            } if self.graph.sheet_id(sheet).is_none() => {
7760                Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
7761            }
7762            ASTNodeType::Reference {
7763                reference:
7764                    ReferenceType::External(_)
7765                    | ReferenceType::Cell3D { .. }
7766                    | ReferenceType::Range3D { .. },
7767                ..
7768            } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
7769            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
7770        }
7771    }
7772
7773    fn target_planning_snapshot(
7774        &mut self,
7775        ast: &ASTNode,
7776        planning_requests: &mut BTreeSet<(String, String, usize)>,
7777    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7778        #[cfg(test)]
7779        if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
7780            return Err(Self::preparation_stale(
7781                formualizer_common::PreparationStaleReason::Semantic,
7782                "injected target semantic preparation movement",
7783            ));
7784        }
7785        #[cfg(test)]
7786        if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
7787            hook();
7788        }
7789        let mut requests = Vec::new();
7790        Self::collect_planning_function_requests(ast, &mut requests);
7791        requests.sort();
7792        requests.dedup();
7793        planning_requests.extend(requests.iter().cloned());
7794        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7795            &self.resolver,
7796            requests,
7797        )
7798        .map_err(|error| {
7799            ExcelError::new(ExcelErrorKind::Value)
7800                .with_message(format!("target planning snapshot unavailable: {error:?}"))
7801        })
7802    }
7803
7804    fn target_planning_snapshot_stale_reason(
7805        snapshot: &crate::function_registry::RegistryPlanningSnapshot,
7806        assumptions: &crate::engine::PreparationRevision,
7807    ) -> Option<formualizer_common::PreparationStaleReason> {
7808        if snapshot
7809            .provider_revision()
7810            .is_some_and(|revision| Some(revision) != assumptions.provider)
7811        {
7812            Some(formualizer_common::PreparationStaleReason::Provider)
7813        } else if snapshot.epoch() != assumptions.semantic
7814            && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
7815        {
7816            Some(formualizer_common::PreparationStaleReason::Semantic)
7817        } else {
7818            None
7819        }
7820    }
7821
7822    fn widen_target_preparation(
7823        policy: crate::engine::OpaquePreparePolicy,
7824        scope: &mut crate::engine::PrepareScope,
7825        reasons: &mut Vec<crate::engine::OpaqueReason>,
7826        reason: crate::engine::OpaqueReason,
7827    ) -> Result<bool, ExcelError> {
7828        if policy == crate::engine::OpaquePreparePolicy::Error {
7829            return Err(ExcelError::new(ExcelErrorKind::NImpl)
7830                .with_message(format!("opaque target preparation semantics: {reason:?}")));
7831        }
7832        if !reasons.contains(&reason) {
7833            reasons.push(reason);
7834        }
7835        if !matches!(scope, crate::engine::PrepareScope::Workbook) {
7836            *scope = crate::engine::PrepareScope::Workbook;
7837            Ok(true)
7838        } else {
7839            Ok(false)
7840        }
7841    }
7842
7843    fn widen_target_preparation_to_sheet(
7844        policy: crate::engine::OpaquePreparePolicy,
7845        scope: &mut crate::engine::PrepareScope,
7846        reasons: &mut Vec<crate::engine::OpaqueReason>,
7847        reason: crate::engine::OpaqueReason,
7848        sheet: &str,
7849    ) -> Result<bool, ExcelError> {
7850        if policy == crate::engine::OpaquePreparePolicy::Error {
7851            return Err(ExcelError::new(ExcelErrorKind::NImpl)
7852                .with_message(format!("opaque target preparation semantics: {reason:?}")));
7853        }
7854        if !reasons.contains(&reason) {
7855            reasons.push(reason);
7856        }
7857        match scope {
7858            crate::engine::PrepareScope::Exact => {
7859                *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
7860                Ok(true)
7861            }
7862            crate::engine::PrepareScope::Sheets(sheets) => {
7863                if sheets.iter().any(|candidate| candidate == sheet) {
7864                    Ok(false)
7865                } else {
7866                    sheets.push(sheet.to_string());
7867                    sheets.sort();
7868                    Ok(true)
7869                }
7870            }
7871            crate::engine::PrepareScope::Workbook => Ok(false),
7872        }
7873    }
7874
7875    fn ast_has_proven_sheet_local_dynamic(
7876        ast: &ASTNode,
7877        provider: &dyn crate::traits::FunctionProvider,
7878    ) -> bool {
7879        fn classify(ast: &ASTNode, provider: &dyn crate::traits::FunctionProvider) -> (bool, bool) {
7880            match &ast.node_type {
7881                ASTNodeType::Function { name, args } => {
7882                    let Some(function) = provider.get_function_for_planning("", name) else {
7883                        return (false, false);
7884                    };
7885                    let caps = function.caps();
7886                    let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
7887                        || caps.contains(FnCaps::RETURNS_REFERENCE);
7888                    let canonical = name.rsplit('.').next().unwrap_or(name);
7889                    if dynamic
7890                        && !canonical.eq_ignore_ascii_case("OFFSET")
7891                        && !canonical.eq_ignore_ascii_case("INDEX")
7892                    {
7893                        return (false, true);
7894                    }
7895                    let mut has_dynamic = dynamic;
7896                    for arg in args {
7897                        let (safe, child_dynamic) = classify(arg, provider);
7898                        if !safe {
7899                            return (false, has_dynamic || child_dynamic);
7900                        }
7901                        has_dynamic |= child_dynamic;
7902                    }
7903                    (true, has_dynamic)
7904                }
7905                ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
7906                ASTNodeType::BinaryOp { left, right, .. } => {
7907                    let (left_safe, left_dynamic) = classify(left, provider);
7908                    let (right_safe, right_dynamic) = classify(right, provider);
7909                    (left_safe && right_safe, left_dynamic || right_dynamic)
7910                }
7911                ASTNodeType::Array(rows) => {
7912                    let mut has_dynamic = false;
7913                    for item in rows.iter().flatten() {
7914                        let (safe, child_dynamic) = classify(item, provider);
7915                        if !safe {
7916                            return (false, has_dynamic || child_dynamic);
7917                        }
7918                        has_dynamic |= child_dynamic;
7919                    }
7920                    (true, has_dynamic)
7921                }
7922                ASTNodeType::Reference {
7923                    reference:
7924                        ReferenceType::Cell { sheet: None, .. }
7925                        | ReferenceType::Range { sheet: None, .. },
7926                    ..
7927                }
7928                | ASTNodeType::Literal(_)
7929                | ASTNodeType::Omitted => (true, false),
7930                ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
7931            }
7932        }
7933
7934        let (safe, dynamic) = classify(ast, provider);
7935        safe && dynamic
7936    }
7937
7938    fn table_selection_region(
7939        &self,
7940        entry: &crate::engine::graph::TableEntry,
7941        selection: &crate::engine::TableSelection,
7942    ) -> Result<PreparationRegion, ExcelError> {
7943        let mut start_row = entry.range.start.coord.row() + 1;
7944        let mut end_row = entry.range.end.coord.row() + 1;
7945        let mut start_col = entry.range.start.coord.col() + 1;
7946        let mut end_col = entry.range.end.coord.col() + 1;
7947        match selection {
7948            crate::engine::TableSelection::Whole => {}
7949            crate::engine::TableSelection::Headers => {
7950                if !entry.header_row {
7951                    return Err(ExcelError::new(ExcelErrorKind::Value)
7952                        .with_message(format!("table {} has no header row", entry.name)));
7953                }
7954                end_row = start_row;
7955            }
7956            crate::engine::TableSelection::Data => {
7957                if entry.header_row {
7958                    start_row = start_row.saturating_add(1);
7959                }
7960                if entry.totals_row {
7961                    end_row = end_row.saturating_sub(1);
7962                }
7963            }
7964            crate::engine::TableSelection::Totals => {
7965                if !entry.totals_row {
7966                    return Err(ExcelError::new(ExcelErrorKind::Value)
7967                        .with_message(format!("table {} has no totals row", entry.name)));
7968                }
7969                start_row = end_row;
7970            }
7971            crate::engine::TableSelection::Column(column) => {
7972                let index = entry.col_index(column).ok_or_else(|| {
7973                    ExcelError::new(ExcelErrorKind::Name)
7974                        .with_message(format!("unknown table column: {column}"))
7975                })?;
7976                start_col = start_col.saturating_add(index as u32);
7977                end_col = start_col;
7978            }
7979            crate::engine::TableSelection::Columns { start, end } => {
7980                let first = entry.col_index(start).ok_or_else(|| {
7981                    ExcelError::new(ExcelErrorKind::Name)
7982                        .with_message(format!("unknown table column: {start}"))
7983                })?;
7984                let last = entry.col_index(end).ok_or_else(|| {
7985                    ExcelError::new(ExcelErrorKind::Name)
7986                        .with_message(format!("unknown table column: {end}"))
7987                })?;
7988                if first > last {
7989                    return Err(ExcelError::new(ExcelErrorKind::Value)
7990                        .with_message("table column selection is reversed"));
7991                }
7992                start_col = start_col.saturating_add(first as u32);
7993                end_col = entry
7994                    .range
7995                    .start
7996                    .coord
7997                    .col()
7998                    .saturating_add(last as u32)
7999                    .saturating_add(1);
8000            }
8001        }
8002        if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
8003            start_row = entry.range.start.coord.row() + 1;
8004            end_row = start_row;
8005        }
8006        if start_row > end_row || start_col > end_col {
8007            return Err(
8008                ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
8009            );
8010        }
8011        Ok(PreparationRegion {
8012            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
8013            sheet_id: entry.sheet_id(),
8014            start_row,
8015            start_col,
8016            end_row,
8017            end_col,
8018        })
8019    }
8020
8021    #[cfg(test)]
8022    pub(crate) fn reset_target_root_dedup_probes_for_test() {
8023        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
8024    }
8025
8026    #[cfg(test)]
8027    pub(crate) fn target_root_dedup_probes_for_test() -> usize {
8028        TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
8029    }
8030
8031    pub(crate) fn resolve_target_producers(
8032        &mut self,
8033        targets: &[crate::engine::EvaluationTarget],
8034    ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
8035        use crate::engine::target_preparation::TargetProducer;
8036
8037        let request_id = self
8038            .active_evaluation_resource_request
8039            .as_ref()
8040            .map(|request| request.request_id);
8041        let mut roots = OrderedTargetProducers::with_capacity(targets.len())
8042            .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
8043        let resolve_region = |engine: &mut Self,
8044                              region: Region,
8045                              value_only: Option<CellRef>,
8046                              roots: &mut OrderedTargetProducers|
8047         -> Result<(), ExcelError> {
8048            let before = roots.len();
8049            for anchor in engine.graph.spill_anchors_in_region(
8050                region.sheet_id(),
8051                region.axis_ranges().0.query_bounds().0,
8052                region.axis_ranges().1.query_bounds().0,
8053                region.axis_ranges().0.query_bounds().1,
8054                region.axis_ranges().1.query_bounds().1,
8055            ) {
8056                roots
8057                    .push(TargetProducer::Legacy(anchor))
8058                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8059            }
8060            for vertex in engine.graph.vertices_in_region(
8061                region.sheet_id(),
8062                region.axis_ranges().0.query_bounds().0,
8063                region.axis_ranges().0.query_bounds().1,
8064                region.axis_ranges().1.query_bounds().0,
8065                region.axis_ranges().1.query_bounds().1,
8066            ) {
8067                let vertex = engine
8068                    .graph
8069                    .get_cell_ref(vertex)
8070                    .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
8071                    .unwrap_or(vertex);
8072                // `vertices_in_region` is a sheet-index query and a sheet index holds only
8073                // grid-addressed vertices, so a region can never yield a symbol: names,
8074                // tables and external sources have no position for a region to cover.
8075                // Symbol roots come from the by-name lookups below instead.
8076                if matches!(
8077                    engine.graph.get_vertex_kind(vertex),
8078                    VertexKind::FormulaScalar | VertexKind::FormulaArray
8079                ) {
8080                    roots
8081                        .push(TargetProducer::Legacy(vertex))
8082                        .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8083                }
8084            }
8085            if roots.len() == before
8086                && let Some(cell) = value_only
8087            {
8088                roots
8089                    .push(TargetProducer::ValueOnly(cell))
8090                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8091            }
8092            Ok(())
8093        };
8094
8095        for target in targets {
8096            match target {
8097                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8098                    if *row == 0 || *col == 0 {
8099                        return Err(ExcelError::new(ExcelErrorKind::Ref)
8100                            .with_message("target cell coordinates are one-based"));
8101                    }
8102                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8103                        ExcelError::new(ExcelErrorKind::Ref)
8104                            .with_message(format!("target sheet not found: {sheet}"))
8105                    })?;
8106                    let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
8107                    resolve_region(
8108                        self,
8109                        Region::point(sheet_id, *row - 1, *col - 1),
8110                        Some(cell),
8111                        &mut roots,
8112                    )?;
8113                }
8114                crate::engine::EvaluationTarget::Range(range) => {
8115                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8116                        ExcelError::new(ExcelErrorKind::Ref)
8117                            .with_message(format!("target sheet not found: {}", range.sheet))
8118                    })?;
8119                    resolve_region(
8120                        self,
8121                        Region::rect(
8122                            sheet_id,
8123                            range.start_row - 1,
8124                            range.end_row - 1,
8125                            range.start_col - 1,
8126                            range.end_col - 1,
8127                        ),
8128                        None,
8129                        &mut roots,
8130                    )?;
8131                }
8132                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8133                    let scope = self.name_query_scope(scope_sheet.as_deref())?;
8134                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
8135                        roots
8136                            .push(TargetProducer::Symbol(entry.vertex))
8137                            .map_err(|_| {
8138                                target_root_allocation_error(roots.len() + 1, request_id)
8139                            })?;
8140                    }
8141                }
8142                crate::engine::EvaluationTarget::Table { name, .. } => {
8143                    if let Some(entry) = self.graph.resolve_table_entry(name) {
8144                        roots
8145                            .push(TargetProducer::Symbol(entry.vertex))
8146                            .map_err(|_| {
8147                                target_root_allocation_error(roots.len() + 1, request_id)
8148                            })?;
8149                    }
8150                }
8151            }
8152        }
8153        Ok(roots.into_vec())
8154    }
8155
8156    /// Transactionally prepare the complete ordinary staged demand closure for typed targets.
8157    ///
8158    /// This method prepares graph topology only. It does not evaluate target values.
8159    pub fn prepare_graph_for_targets(
8160        &mut self,
8161        targets: &[crate::engine::EvaluationTarget],
8162        options: crate::engine::TargetEvalOptions<'_>,
8163    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8164        let previous_budgets = self.evaluation_resource_budgets.clone();
8165        let diagnostics_len = self.formula_parse_diagnostics.len();
8166        let previous_report = self.last_formula_ingest_report.clone();
8167        if let Some(budgets) = options.budgets {
8168            self.evaluation_resource_budgets = budgets.clone();
8169        }
8170        let previous_graph_budget_override = self
8171            .graph
8172            .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
8173        // Hoist the call's cancellation onto the engine for its duration, so the
8174        // preparation checkpoints observe it. This is a standalone entry point, so
8175        // the previous value is restored rather than cleared.
8176        let previous_cancel = self.active_cancel_flag.take();
8177        self.active_cancel_flag = options.cancel.clone();
8178        let result = self.observe_evaluation_resource_request(
8179            EvaluationRequestKind::TargetPreparation,
8180            |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
8181        );
8182        self.active_cancel_flag = previous_cancel;
8183        self.graph
8184            .set_admission_budget_override(previous_graph_budget_override);
8185        self.evaluation_resource_budgets = previous_budgets;
8186        if result.is_err() {
8187            self.formula_parse_diagnostics.truncate(diagnostics_len);
8188            self.last_formula_ingest_report = previous_report;
8189        }
8190        result
8191    }
8192
8193    fn prepare_graph_for_targets_unobserved(
8194        &mut self,
8195        targets: &[crate::engine::EvaluationTarget],
8196        options: &crate::engine::TargetEvalOptions<'_>,
8197    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8198        let scratch_checkpoint = self
8199            .active_resource_ledger
8200            .as_ref()
8201            .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
8202        let result = self.prepare_graph_for_targets_transaction(targets, options);
8203        let release = self
8204            .active_resource_ledger
8205            .as_mut()
8206            .map_or(Ok(()), |ledger| {
8207                ledger.release_scratch_to(scratch_checkpoint)
8208            });
8209        match (result, release) {
8210            (result, Ok(())) => result,
8211            (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
8212        }
8213    }
8214
8215    fn prepare_graph_for_targets_transaction(
8216        &mut self,
8217        targets: &[crate::engine::EvaluationTarget],
8218        options: &crate::engine::TargetEvalOptions<'_>,
8219    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8220        use crate::engine::{
8221            OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
8222            TableSelection,
8223        };
8224
8225        self.target_preparation_checkpoint(options.deadline, 0)?;
8226        self.observe_function_semantic_epoch()?;
8227        let assumptions = self.preparation_revisions();
8228        let ledger_at_start = self
8229            .active_resource_ledger
8230            .as_ref()
8231            .map(|ledger| ledger.snapshot());
8232        let diagnostics_len = self.formula_parse_diagnostics.len();
8233        let report_len = self.last_formula_ingest_report.clone();
8234        let request_id = options.request_id.or_else(|| {
8235            self.active_evaluation_resource_request
8236                .as_ref()
8237                .map(|stats| stats.request_id)
8238        });
8239
8240        let mut scope = PrepareScope::Exact;
8241        let mut reasons = Vec::new();
8242        let mut regions = VecDeque::new();
8243        let mut deferred_shared_regions = VecDeque::new();
8244        let mut normalized = Vec::with_capacity(targets.len());
8245        let mut symbol_vertices = VecDeque::new();
8246        for target in targets {
8247            self.target_preparation_checkpoint(options.deadline, 1)?;
8248            match target {
8249                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8250                    if *row == 0 || *col == 0 {
8251                        return Err(ExcelError::new(ExcelErrorKind::Ref)
8252                            .with_message("target cell coordinates are one-based"));
8253                    }
8254                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8255                        ExcelError::new(ExcelErrorKind::Ref)
8256                            .with_message(format!("target sheet not found: {sheet}"))
8257                    })?;
8258                    regions.push_back(PreparationRegion {
8259                        sheet: sheet.clone(),
8260                        sheet_id,
8261                        start_row: *row,
8262                        start_col: *col,
8263                        end_row: *row,
8264                        end_col: *col,
8265                    });
8266                    normalized.push(target.clone());
8267                }
8268                crate::engine::EvaluationTarget::Range(range) => {
8269                    if range.start_row == 0
8270                        || range.start_col == 0
8271                        || range.end_row < range.start_row
8272                        || range.end_col < range.start_col
8273                    {
8274                        return Err(ExcelError::new(ExcelErrorKind::Ref)
8275                            .with_message("invalid target range"));
8276                    }
8277                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8278                        ExcelError::new(ExcelErrorKind::Ref)
8279                            .with_message(format!("target sheet not found: {}", range.sheet))
8280                    })?;
8281                    regions.push_back(PreparationRegion {
8282                        sheet: range.sheet.clone(),
8283                        sheet_id,
8284                        start_row: range.start_row,
8285                        start_col: range.start_col,
8286                        end_row: range.end_row,
8287                        end_col: range.end_col,
8288                    });
8289                    normalized.push(target.clone());
8290                }
8291                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8292                    let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
8293                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
8294                        symbol_vertices.push_back(entry.vertex);
8295                    } else {
8296                        Self::widen_target_preparation(
8297                            options.opaque_policy,
8298                            &mut scope,
8299                            &mut reasons,
8300                            OpaqueReason::UnresolvedName,
8301                        )?;
8302                    }
8303                    normalized.push(target.clone());
8304                }
8305                crate::engine::EvaluationTarget::Table { name, selection } => {
8306                    if let Some(entry) = self.graph.resolve_table_entry(name) {
8307                        let region = self.table_selection_region(entry, selection)?;
8308                        symbol_vertices.push_back(entry.vertex);
8309                        regions.push_back(region);
8310                    } else {
8311                        Self::widen_target_preparation(
8312                            options.opaque_policy,
8313                            &mut scope,
8314                            &mut reasons,
8315                            OpaqueReason::UnresolvedTable,
8316                        )?;
8317                    }
8318                    normalized.push(target.clone());
8319                }
8320            }
8321        }
8322
8323        let mut visited_regions = FxHashSet::default();
8324        let mut visited_vertices = FxHashSet::default();
8325        let mut selected = FxHashSet::default();
8326        let mut prepared = Vec::new();
8327        let mut pending_diagnostics = Vec::new();
8328        let mut planning_requests = BTreeSet::new();
8329        let mut selected_cells = Vec::new();
8330        let mut workbook_seeded = false;
8331        let mut sheet_scope_seeded = BTreeSet::new();
8332        let mut indexed_query_sheets = FxHashSet::default();
8333        let mut discovery_scratch_reserved = 0u64;
8334        let mut package_encountered = false;
8335        let mut selected_package_sheets = FxHashSet::default();
8336        let mut selected_package_points: BTreeMap<String, BTreeSet<(u32, u32)>> = BTreeMap::new();
8337        let mut prepared_packages: Vec<PreparedTargetSourcePackage> = Vec::new();
8338        let authoritative_with_ordinary = false;
8339        let has_unknown_package_sheet = self
8340            .staged_formula_index
8341            .package_sheets()
8342            .any(|sheet| self.graph.sheet_id(sheet).is_none());
8343
8344        loop {
8345            if let PrepareScope::Sheets(sheets) = &scope {
8346                for sheet in sheets.clone() {
8347                    if !sheet_scope_seeded.insert(sheet.clone()) {
8348                        continue;
8349                    }
8350                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8351                        package_encountered = true;
8352                        Self::widen_target_preparation(
8353                            options.opaque_policy,
8354                            &mut scope,
8355                            &mut reasons,
8356                            OpaqueReason::UnsupportedSourceSemantics,
8357                        )?;
8358                        continue;
8359                    };
8360                    for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
8361                        self.target_preparation_checkpoint(options.deadline, 1)?;
8362                        regions.push_back(PreparationRegion {
8363                            sheet: sheet.clone(),
8364                            sheet_id,
8365                            start_row: lease.row,
8366                            start_col: lease.col,
8367                            end_row: lease.row,
8368                            end_col: lease.col,
8369                        });
8370                    }
8371                    if self
8372                        .staged_formula_index
8373                        .package_lease_for_sheet(&sheet)
8374                        .is_some()
8375                    {
8376                        regions.push_back(PreparationRegion {
8377                            sheet: sheet.clone(),
8378                            sheet_id,
8379                            start_row: 1,
8380                            start_col: 1,
8381                            end_row: self.workbook_load_limits.max_sheet_rows,
8382                            end_col: self.workbook_load_limits.max_sheet_cols,
8383                        });
8384                    }
8385                }
8386            }
8387            if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
8388                workbook_seeded = true;
8389                for (sheet, lease) in self.staged_formula_index.all_leases() {
8390                    self.target_preparation_checkpoint(options.deadline, 1)?;
8391                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8392                        package_encountered = true;
8393                        Self::widen_target_preparation(
8394                            options.opaque_policy,
8395                            &mut scope,
8396                            &mut reasons,
8397                            OpaqueReason::UnsupportedSourceSemantics,
8398                        )?;
8399                        continue;
8400                    };
8401                    regions.push_back(PreparationRegion {
8402                        sheet,
8403                        sheet_id,
8404                        start_row: lease.row,
8405                        start_col: lease.col,
8406                        end_row: lease.row,
8407                        end_col: lease.col,
8408                    });
8409                }
8410                let package_sheets = self
8411                    .staged_formula_index
8412                    .package_sheets()
8413                    .map(str::to_string)
8414                    .collect::<Vec<_>>();
8415                for sheet in package_sheets {
8416                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8417                        package_encountered = true;
8418                        Self::widen_target_preparation(
8419                            options.opaque_policy,
8420                            &mut scope,
8421                            &mut reasons,
8422                            OpaqueReason::UnsupportedSourceSemantics,
8423                        )?;
8424                        continue;
8425                    };
8426                    regions.push_back(PreparationRegion {
8427                        sheet,
8428                        sheet_id,
8429                        start_row: 1,
8430                        start_col: 1,
8431                        end_row: self.workbook_load_limits.max_sheet_rows,
8432                        end_col: self.workbook_load_limits.max_sheet_cols,
8433                    });
8434                }
8435            }
8436
8437            // Finish known ordinary/name dependency discovery before expanding
8438            // partial shared demands. A queued SUM may complete those families.
8439            let allow_partial_shared = regions.is_empty() && symbol_vertices.is_empty();
8440            let next_region = if allow_partial_shared {
8441                deferred_shared_regions.pop_front()
8442            } else {
8443                regions.pop_front()
8444            };
8445            let Some(region) = next_region else {
8446                if let Some(vertex) = symbol_vertices.pop_front() {
8447                    self.target_preparation_checkpoint(options.deadline, 1)?;
8448                    if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
8449                        continue;
8450                    }
8451                    let vertex_is_dynamic = self.graph.is_dynamic(vertex);
8452                    if let Some(ast) = self.graph.get_formula(vertex) {
8453                        let snapshot =
8454                            self.target_planning_snapshot(&ast, &mut planning_requests)?;
8455                        if let Some(reason) =
8456                            Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
8457                        {
8458                            return Err(Self::preparation_stale(
8459                                reason,
8460                                "target planning snapshot became stale during discovery",
8461                            ));
8462                        }
8463                        let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
8464                        if let Some(reason) =
8465                            opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
8466                        {
8467                            if reason == OpaqueReason::DynamicReference
8468                                && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
8469                            {
8470                                let sheet = self.graph.get_vertex_sheet_id(vertex);
8471                                let sheet = self.graph.sheet_name(sheet).to_string();
8472                                Self::widen_target_preparation_to_sheet(
8473                                    options.opaque_policy,
8474                                    &mut scope,
8475                                    &mut reasons,
8476                                    reason,
8477                                    &sheet,
8478                                )?;
8479                            } else {
8480                                Self::widen_target_preparation(
8481                                    options.opaque_policy,
8482                                    &mut scope,
8483                                    &mut reasons,
8484                                    reason,
8485                                )?;
8486                            }
8487                        }
8488                    } else if vertex_is_dynamic {
8489                        Self::widen_target_preparation(
8490                            options.opaque_policy,
8491                            &mut scope,
8492                            &mut reasons,
8493                            OpaqueReason::DynamicReference,
8494                        )?;
8495                    }
8496                    if let Some(anchor) = self
8497                        .graph
8498                        .get_cell_ref(vertex)
8499                        .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
8500                    {
8501                        symbol_vertices.push_back(anchor);
8502                    }
8503                    if let Some(cell) = self.graph.get_cell_ref(vertex) {
8504                        let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
8505                        regions.push_back(PreparationRegion {
8506                            sheet,
8507                            sheet_id: cell.sheet_id,
8508                            start_row: cell.coord.row() + 1,
8509                            start_col: cell.coord.col() + 1,
8510                            end_row: cell.coord.row() + 1,
8511                            end_col: cell.coord.col() + 1,
8512                        });
8513                    }
8514                    // The formula's direct precedents, from the authority:
8515                    // cells and ranges become regions, symbol rows (names,
8516                    // tables, sources) their vertices.
8517                    match self.graph.authority_vertex_precedents(vertex) {
8518                        Some(precedents) => {
8519                            for (sheet_id, rect) in precedents {
8520                                self.target_preparation_checkpoint(options.deadline, 1)?;
8521                                if sheet_id == crate::engine::authority::geom::SYMBOL_SHEET {
8522                                    for slot in rect.r0..=rect.r1 {
8523                                        if let Some(symbol) =
8524                                            self.graph.authority_host().symbols().vertex(slot)
8525                                        {
8526                                            symbol_vertices.push_back(symbol);
8527                                        }
8528                                    }
8529                                    continue;
8530                                }
8531                                let sheet = self.graph.sheet_name(sheet_id).to_string();
8532                                regions.push_back(PreparationRegion {
8533                                    sheet,
8534                                    sheet_id,
8535                                    start_row: rect.r0 + 1,
8536                                    start_col: rect.c0 + 1,
8537                                    end_row: (rect.r1 + 1)
8538                                        .min(self.workbook_load_limits.max_sheet_rows),
8539                                    end_col: (rect.c1 + 1)
8540                                        .min(self.workbook_load_limits.max_sheet_cols),
8541                                });
8542                            }
8543                        }
8544                        // The authority cannot answer (failed host): widen.
8545                        None => {
8546                            if Self::widen_target_preparation(
8547                                options.opaque_policy,
8548                                &mut scope,
8549                                &mut reasons,
8550                                OpaqueReason::UnresolvedCrossSheetBinding,
8551                            )? {
8552                                continue;
8553                            }
8554                        }
8555                    }
8556                    if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
8557                        match &name.definition {
8558                            NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
8559                                sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
8560                                sheet_id: cell.sheet_id,
8561                                start_row: cell.coord.row() + 1,
8562                                start_col: cell.coord.col() + 1,
8563                                end_row: cell.coord.row() + 1,
8564                                end_col: cell.coord.col() + 1,
8565                            }),
8566                            NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
8567                                sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
8568                                sheet_id: range.start.sheet_id,
8569                                start_row: range.start.coord.row() + 1,
8570                                start_col: range.start.coord.col() + 1,
8571                                end_row: range.end.coord.row() + 1,
8572                                end_col: range.end.coord.col() + 1,
8573                            }),
8574                            NamedDefinition::Formula {
8575                                ast,
8576                                dependencies,
8577                                range_deps,
8578                            } => {
8579                                let snapshot =
8580                                    self.target_planning_snapshot(ast, &mut planning_requests)?;
8581                                if let Some(reason) = Self::target_planning_snapshot_stale_reason(
8582                                    &snapshot,
8583                                    &assumptions,
8584                                ) {
8585                                    return Err(Self::preparation_stale(
8586                                        reason,
8587                                        "target planning snapshot became stale during discovery",
8588                                    ));
8589                                }
8590                                if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
8591                                    Self::widen_target_preparation(
8592                                        options.opaque_policy,
8593                                        &mut scope,
8594                                        &mut reasons,
8595                                        reason,
8596                                    )?;
8597                                }
8598                                for dependency in dependencies {
8599                                    self.target_preparation_checkpoint(options.deadline, 1)?;
8600                                    symbol_vertices.push_back(*dependency);
8601                                }
8602                                for range in range_deps {
8603                                    self.target_preparation_checkpoint(options.deadline, 1)?;
8604                                    // `Current` is the sheet this name's formula
8605                                    // was interpreted on, which is the sheet its
8606                                    // vertex is placed on -- the same derivation
8607                                    // the formula-vertex arm above uses. It is
8608                                    // never the workbook default sheet, and an
8609                                    // unresolvable `Name` widens instead of
8610                                    // silently landing on some other sheet.
8611                                    let context_sheet = self.graph.get_vertex_sheet_id(vertex);
8612                                    let Ok(sheet_id) =
8613                                        self.resolve_sheet_locator(&range.sheet, context_sheet)
8614                                    else {
8615                                        if Self::widen_target_preparation(
8616                                            options.opaque_policy,
8617                                            &mut scope,
8618                                            &mut reasons,
8619                                            OpaqueReason::UnresolvedCrossSheetBinding,
8620                                        )? {
8621                                            break;
8622                                        }
8623                                        continue;
8624                                    };
8625                                    regions.push_back(PreparationRegion {
8626                                        sheet: self.graph.sheet_name(sheet_id).to_string(),
8627                                        sheet_id,
8628                                        start_row: range
8629                                            .start_row
8630                                            .map_or(1, |bound| bound.index + 1),
8631                                        start_col: range
8632                                            .start_col
8633                                            .map_or(1, |bound| bound.index + 1),
8634                                        end_row: range.end_row.map_or(
8635                                            self.workbook_load_limits.max_sheet_rows,
8636                                            |bound| bound.index + 1,
8637                                        ),
8638                                        end_col: range.end_col.map_or(
8639                                            self.workbook_load_limits.max_sheet_cols,
8640                                            |bound| bound.index + 1,
8641                                        ),
8642                                    });
8643                                }
8644                            }
8645                            NamedDefinition::Literal(_) => {}
8646                        }
8647                    }
8648                    if let Some(table) = self.graph.table_by_vertex(vertex) {
8649                        regions
8650                            .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
8651                    }
8652                    continue;
8653                }
8654                break;
8655            };
8656
8657            self.target_preparation_checkpoint(options.deadline, 1)?;
8658            if !visited_regions.insert(region.clone()) && !allow_partial_shared {
8659                continue;
8660            }
8661            if let PrepareScope::Sheets(sheets) = &mut scope
8662                && !sheets.iter().any(|sheet| sheet == &region.sheet)
8663            {
8664                sheets.push(region.sheet.clone());
8665                sheets.sort();
8666            }
8667            let package_match = self.staged_formula_index.package_for_region(
8668                &region.sheet,
8669                region.start_row,
8670                region.start_col,
8671                region.end_row,
8672                region.end_col,
8673            );
8674            let package_lease = match package_match {
8675                Some(Ok(lease)) => Some(lease),
8676                Some(Err(()))
8677                    if region.start_row == 1
8678                        && region.start_col == 1
8679                        && region.end_row == self.workbook_load_limits.max_sheet_rows
8680                        && region.end_col == self.workbook_load_limits.max_sheet_cols =>
8681                {
8682                    self.staged_formula_index
8683                        .package_lease_for_sheet(&region.sheet)
8684                }
8685                Some(Err(())) => {
8686                    Self::widen_target_preparation(
8687                        options.opaque_policy,
8688                        &mut scope,
8689                        &mut reasons,
8690                        OpaqueReason::DeferredSourcePackage,
8691                    )?;
8692                    None
8693                }
8694                None => None,
8695            };
8696            let compatibility_before_package_replay = package_lease.is_some()
8697                && (authoritative_with_ordinary || has_unknown_package_sheet);
8698            let package_lease = if compatibility_before_package_replay {
8699                package_encountered = true;
8700                Self::widen_target_preparation(
8701                    options.opaque_policy,
8702                    &mut scope,
8703                    &mut reasons,
8704                    OpaqueReason::UnsupportedSourceSemantics,
8705                )?;
8706                None
8707            } else {
8708                package_lease
8709            };
8710            if let Some(package_lease) = package_lease
8711                && !selected_package_sheets.contains(&region.sheet)
8712            {
8713                self.target_preparation_checkpoint(options.deadline, 1)?;
8714                let mut points = self.staged_formula_index.package_points_in_region(
8715                    &region.sheet,
8716                    region.start_row,
8717                    region.start_col,
8718                    region.end_row,
8719                    region.end_col,
8720                );
8721                if let Some(selected) = selected_package_points.get(&region.sheet) {
8722                    points.retain(|point| !selected.contains(point));
8723                }
8724
8725                if let Some(selected) = selected_package_points.get(&region.sheet) {
8726                    points.retain(|point| !selected.contains(point));
8727                }
8728                let permit_partial = allow_partial_shared
8729                    || (points.len() == 1
8730                        && regions.is_empty()
8731                        && symbol_vertices.is_empty()
8732                        && deferred_shared_regions.is_empty());
8733                let partial = self.prepare_target_exact_source_selection(
8734                    &region.sheet,
8735                    package_lease,
8736                    points,
8737                    selected_package_points
8738                        .get(&region.sheet)
8739                        .unwrap_or(&BTreeSet::new()),
8740                    permit_partial,
8741                    options.deadline,
8742                    &mut discovery_scratch_reserved,
8743                )?;
8744                let mut package = if let Some(mut package) = partial {
8745                    if package.deferred_shared {
8746                        deferred_shared_regions.push_back(region.clone());
8747                    }
8748                    // A later range can complete a family touched earlier in this
8749                    // request. Retire only that family's earlier legacy proposals;
8750                    // nothing has been published and its dependencies remain demanded.
8751                    if !package.direct_domains.is_empty() {
8752                        for prior in prepared_packages
8753                            .iter_mut()
8754                            .filter(|prior| prior.sheet == region.sheet)
8755                        {
8756                            prior
8757                                .replay_records
8758                                .retain(|record| !package.direct_contains(record.row, record.col));
8759                            prior
8760                                .legacy
8761                                .retain(|(row, col, _, _)| !package.direct_contains(*row, *col));
8762                            if let Some(points) = prior.selected_points.as_mut() {
8763                                points.retain(|&(row, col)| !package.direct_contains(row, col));
8764                            }
8765                        }
8766                    }
8767                    if selected_package_points.contains_key(&region.sheet) {
8768                        package.source_report = Default::default();
8769                    }
8770                    let points = package.selected_points.as_ref().unwrap();
8771                    if !points.is_empty() {
8772                        selected_package_points
8773                            .entry(region.sheet.clone())
8774                            .or_default()
8775                            .extend(points.iter().copied());
8776                    }
8777                    package
8778                } else {
8779                    selected_package_sheets.insert(region.sheet.clone());
8780                    self.prepare_target_source_package(
8781                        &region.sheet,
8782                        package_lease,
8783                        options.deadline,
8784                    )?
8785                };
8786                if package
8787                    .selected_points
8788                    .as_ref()
8789                    .is_none_or(|points| !points.is_empty())
8790                {
8791                    let mut final_fallback = BTreeMap::new();
8792                    for record in package.fallback_records() {
8793                        final_fallback.insert((record.row, record.col), record.clone());
8794                    }
8795                    let batch = self.formula_batch_from_exact_replay(
8796                        &region.sheet,
8797                        final_fallback.into_values(),
8798                    )?;
8799                    for record in batch.formulas {
8800                        self.target_preparation_checkpoint(options.deadline, 1)?;
8801                        let ast = self
8802                            .graph
8803                            .data_store()
8804                            .retrieve_ast(record.ast_id, self.graph.sheet_reg())
8805                            .ok_or_else(|| {
8806                                ExcelError::new(ExcelErrorKind::Value)
8807                                    .with_message("target fallback AST is unavailable")
8808                            })?;
8809                        let snapshot =
8810                            self.target_planning_snapshot(&ast, &mut planning_requests)?;
8811                        if let Some(reason) =
8812                            Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
8813                        {
8814                            return Err(Self::preparation_stale(
8815                                reason,
8816                                "target fallback planning snapshot became stale during discovery",
8817                            ));
8818                        }
8819                        let proven_sheet_local_dynamic =
8820                            Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
8821                        if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
8822                            if reason == OpaqueReason::DynamicReference
8823                                && proven_sheet_local_dynamic
8824                            {
8825                                Self::widen_target_preparation_to_sheet(
8826                                    options.opaque_policy,
8827                                    &mut scope,
8828                                    &mut reasons,
8829                                    reason,
8830                                    &region.sheet,
8831                                )?;
8832                            } else {
8833                                Self::widen_target_preparation(
8834                                    options.opaque_policy,
8835                                    &mut scope,
8836                                    &mut reasons,
8837                                    reason,
8838                                )?;
8839                            }
8840                        }
8841                        let placement = CellRef::new(
8842                            package.sheet_id,
8843                            Coord::from_excel(record.row, record.col, true, true),
8844                        );
8845                        let ingested = self
8846                            .graph
8847                            .ingest_pipeline(&snapshot)
8848                            .enable_function_semantics()
8849                            .ingest_formula(
8850                                FormulaAstInput::RawArena(record.ast_id),
8851                                placement,
8852                                record.formula_text,
8853                            )?;
8854                        if ingested.dep_plan.dynamic {
8855                            if proven_sheet_local_dynamic {
8856                                Self::widen_target_preparation_to_sheet(
8857                                    options.opaque_policy,
8858                                    &mut scope,
8859                                    &mut reasons,
8860                                    OpaqueReason::DynamicReference,
8861                                    &region.sheet,
8862                                )?;
8863                            } else {
8864                                Self::widen_target_preparation(
8865                                    options.opaque_policy,
8866                                    &mut scope,
8867                                    &mut reasons,
8868                                    OpaqueReason::DynamicReference,
8869                                )?;
8870                            }
8871                        }
8872                        for dep in &ingested.dep_plan.direct_cell_deps {
8873                            self.target_preparation_checkpoint(options.deadline, 1)?;
8874                            regions.push_back(PreparationRegion {
8875                                sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
8876                                sheet_id: dep.sheet_id,
8877                                start_row: dep.coord.row().saturating_add(1),
8878                                start_col: dep.coord.col().saturating_add(1),
8879                                end_row: dep.coord.row().saturating_add(1),
8880                                end_col: dep.coord.col().saturating_add(1),
8881                            });
8882                        }
8883                        for range in &ingested.dep_plan.range_deps {
8884                            self.target_preparation_checkpoint(options.deadline, 1)?;
8885                            // `Current` is the sheet the staged package's formula
8886                            // lives on.
8887                            let Ok(dependency_sheet) =
8888                                self.resolve_sheet_locator(&range.sheet, package.sheet_id)
8889                            else {
8890                                Self::widen_target_preparation(
8891                                    options.opaque_policy,
8892                                    &mut scope,
8893                                    &mut reasons,
8894                                    OpaqueReason::UnresolvedCrossSheetBinding,
8895                                )?;
8896                                continue;
8897                            };
8898                            regions.push_back(PreparationRegion {
8899                                sheet: self.graph.sheet_name(dependency_sheet).to_string(),
8900                                sheet_id: dependency_sheet,
8901                                start_row: range.start_row.map_or(1, |bound| bound.index + 1),
8902                                start_col: range.start_col.map_or(1, |bound| bound.index + 1),
8903                                end_row: range
8904                                    .end_row
8905                                    .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
8906                                        bound.index + 1
8907                                    }),
8908                                end_col: range
8909                                    .end_col
8910                                    .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
8911                                        bound.index + 1
8912                                    }),
8913                            });
8914                        }
8915                        for name in ingested
8916                            .dep_plan
8917                            .resolved_named_refs
8918                            .iter()
8919                            .chain(&ingested.dep_plan.named_refs)
8920                        {
8921                            self.target_preparation_checkpoint(options.deadline, 1)?;
8922                            if let Some(entry) =
8923                                self.graph.resolve_name_entry(name, package.sheet_id)
8924                            {
8925                                symbol_vertices.push_back(entry.vertex);
8926                            } else if self.graph.resolve_source_scalar_entry(name).is_none()
8927                                && self.graph.resolve_source_table_entry(name).is_none()
8928                            {
8929                                Self::widen_target_preparation(
8930                                    options.opaque_policy,
8931                                    &mut scope,
8932                                    &mut reasons,
8933                                    OpaqueReason::UnresolvedName,
8934                                )?;
8935                            }
8936                        }
8937                        for table in &ingested.dep_plan.table_refs {
8938                            self.target_preparation_checkpoint(options.deadline, 1)?;
8939                            if let Some(entry) = self.graph.resolve_table_entry(table) {
8940                                symbol_vertices.push_back(entry.vertex);
8941                            } else if self.graph.resolve_source_table_entry(table).is_none() {
8942                                Self::widen_target_preparation(
8943                                    options.opaque_policy,
8944                                    &mut scope,
8945                                    &mut reasons,
8946                                    OpaqueReason::UnresolvedTable,
8947                                )?;
8948                            }
8949                        }
8950                        package.legacy.push((
8951                            record.row,
8952                            record.col,
8953                            ingested.ast_id,
8954                            ingested.dep_plan,
8955                        ));
8956                    }
8957                    prepared_packages.push(package);
8958                }
8959            }
8960            let leases = self.staged_formula_index.leases_in_region(
8961                &region.sheet,
8962                region.start_row,
8963                region.start_col,
8964                region.end_row,
8965                region.end_col,
8966            );
8967            for lease in leases {
8968                self.target_preparation_checkpoint(options.deadline, 1)?;
8969                let sheet_id = self.graph.sheet_id(&region.sheet).ok_or_else(|| {
8970                    ExcelError::new(ExcelErrorKind::Ref)
8971                        .with_message(format!("staged formula sheet not found: {}", region.sheet))
8972                })?;
8973                let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
8974                if !selected.insert(key) {
8975                    continue;
8976                }
8977                let text = self
8978                    .staged_formulas
8979                    .get(&region.sheet)
8980                    .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
8981                    .ok_or_else(|| {
8982                        ExcelError::new(ExcelErrorKind::Value)
8983                            .with_message("staged formula index is stale")
8984                    })?
8985                    .to_string();
8986                let formula = if text.starts_with('=') {
8987                    text.clone()
8988                } else {
8989                    format!("={text}")
8990                };
8991                self.target_preparation_checkpoint(options.deadline, 1)?;
8992                let ast = match formualizer_parse::parser::parse(&formula) {
8993                    Ok(ast) => ast,
8994                    Err(error) => {
8995                        if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
8996                            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8997                                format!(
8998                                    "Formula parse error at {}!{}{}: {error}",
8999                                    region.sheet,
9000                                    col_letters_from_1based(lease.col)
9001                                        .unwrap_or_else(|_| "?".to_string()),
9002                                    lease.row
9003                                ),
9004                            ));
9005                        }
9006                        pending_diagnostics.push(FormulaParseDiagnostic {
9007                            sheet: region.sheet.clone(),
9008                            row: lease.row,
9009                            col: lease.col,
9010                            formula: formula.clone(),
9011                            message: error.to_string(),
9012                            policy: self.config.formula_parse_policy,
9013                        });
9014                        match self.config.formula_parse_policy {
9015                            FormulaParsePolicy::KeepCachedValue => {
9016                                selected_cells.push(
9017                                    formualizer_common::RangeAddress::new(
9018                                        region.sheet.clone(),
9019                                        lease.row,
9020                                        lease.col,
9021                                        lease.row,
9022                                        lease.col,
9023                                    )
9024                                    .expect("selected staged coordinates are valid"),
9025                                );
9026                                prepared.push(PreparedOrdinaryStagedFormula {
9027                                    sheet: region.sheet.clone(),
9028                                    sheet_id,
9029                                    lease,
9030                                    ast_id: None,
9031                                    plan: None,
9032                                });
9033                                continue;
9034                            }
9035                            FormulaParsePolicy::AsText => ASTNode::new(
9036                                ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
9037                                None,
9038                            ),
9039                            FormulaParsePolicy::CoerceToError => ASTNode::new(
9040                                ASTNodeType::Literal(LiteralValue::Error(
9041                                    ExcelError::new(ExcelErrorKind::Error)
9042                                        .with_message(format!("Malformed formula: {error}")),
9043                                )),
9044                                None,
9045                            ),
9046                            FormulaParsePolicy::Strict => unreachable!(),
9047                        }
9048                    }
9049                };
9050                self.target_preparation_checkpoint(options.deadline, 1)?;
9051                let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
9052                self.target_preparation_checkpoint(options.deadline, 1)?;
9053                if let Some(reason) =
9054                    Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9055                {
9056                    return Err(Self::preparation_stale(
9057                        reason,
9058                        "target planning snapshot became stale during discovery",
9059                    ));
9060                }
9061                if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9062                    if reason == OpaqueReason::DynamicReference
9063                        && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
9064                    {
9065                        Self::widen_target_preparation_to_sheet(
9066                            options.opaque_policy,
9067                            &mut scope,
9068                            &mut reasons,
9069                            reason,
9070                            &region.sheet,
9071                        )?;
9072                    } else {
9073                        Self::widen_target_preparation(
9074                            options.opaque_policy,
9075                            &mut scope,
9076                            &mut reasons,
9077                            reason,
9078                        )?;
9079                    }
9080                }
9081                let proven_sheet_local_dynamic =
9082                    Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9083                let placement = CellRef::new(
9084                    sheet_id,
9085                    Coord::from_excel(lease.row, lease.col, true, true),
9086                );
9087                let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
9088                    FormulaAstInput::Tree(ast),
9089                    placement,
9090                    Some(Arc::from(formula)),
9091                )?;
9092                self.target_preparation_checkpoint(options.deadline, 1)?;
9093                if ingested.dep_plan.dynamic {
9094                    if proven_sheet_local_dynamic {
9095                        Self::widen_target_preparation_to_sheet(
9096                            options.opaque_policy,
9097                            &mut scope,
9098                            &mut reasons,
9099                            OpaqueReason::DynamicReference,
9100                            &region.sheet,
9101                        )?;
9102                    } else {
9103                        Self::widen_target_preparation(
9104                            options.opaque_policy,
9105                            &mut scope,
9106                            &mut reasons,
9107                            OpaqueReason::DynamicReference,
9108                        )?;
9109                    }
9110                }
9111                for dep in &ingested.dep_plan.direct_cell_deps {
9112                    self.target_preparation_checkpoint(options.deadline, 1)?;
9113                    regions.push_back(PreparationRegion {
9114                        sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9115                        sheet_id: dep.sheet_id,
9116                        start_row: dep.coord.row() + 1,
9117                        start_col: dep.coord.col() + 1,
9118                        end_row: dep.coord.row() + 1,
9119                        end_col: dep.coord.col() + 1,
9120                    });
9121                }
9122                for range in &ingested.dep_plan.range_deps {
9123                    self.target_preparation_checkpoint(options.deadline, 1)?;
9124                    // `Current` is the sheet the staged formula lives on.
9125                    let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
9126                    else {
9127                        Self::widen_target_preparation(
9128                            options.opaque_policy,
9129                            &mut scope,
9130                            &mut reasons,
9131                            OpaqueReason::UnresolvedCrossSheetBinding,
9132                        )?;
9133                        continue;
9134                    };
9135                    regions.push_back(PreparationRegion {
9136                        sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9137                        sheet_id: dependency_sheet,
9138                        start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9139                        start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9140                        end_row: range
9141                            .end_row
9142                            .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9143                                bound.index + 1
9144                            }),
9145                        end_col: range
9146                            .end_col
9147                            .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9148                                bound.index + 1
9149                            }),
9150                    });
9151                }
9152                for name in ingested
9153                    .dep_plan
9154                    .resolved_named_refs
9155                    .iter()
9156                    .chain(&ingested.dep_plan.named_refs)
9157                {
9158                    self.target_preparation_checkpoint(options.deadline, 1)?;
9159                    if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
9160                        symbol_vertices.push_back(entry.vertex);
9161                    } else if self.graph.resolve_source_scalar_entry(name).is_none()
9162                        && self.graph.resolve_source_table_entry(name).is_none()
9163                    {
9164                        Self::widen_target_preparation(
9165                            options.opaque_policy,
9166                            &mut scope,
9167                            &mut reasons,
9168                            OpaqueReason::UnresolvedName,
9169                        )?;
9170                    }
9171                }
9172                for table in &ingested.dep_plan.table_refs {
9173                    self.target_preparation_checkpoint(options.deadline, 1)?;
9174                    if let Some(entry) = self.graph.resolve_table_entry(table) {
9175                        symbol_vertices.push_back(entry.vertex);
9176                    } else if self.graph.resolve_source_table_entry(table).is_none() {
9177                        Self::widen_target_preparation(
9178                            options.opaque_policy,
9179                            &mut scope,
9180                            &mut reasons,
9181                            OpaqueReason::UnresolvedTable,
9182                        )?;
9183                    }
9184                }
9185                selected_cells.push(
9186                    formualizer_common::RangeAddress::new(
9187                        region.sheet.clone(),
9188                        lease.row,
9189                        lease.col,
9190                        lease.row,
9191                        lease.col,
9192                    )
9193                    .expect("selected staged coordinates are valid"),
9194                );
9195                prepared.push(PreparedOrdinaryStagedFormula {
9196                    sheet: region.sheet.clone(),
9197                    sheet_id,
9198                    lease,
9199                    ast_id: Some(ingested.ast_id),
9200                    plan: Some(ingested.dep_plan),
9201                });
9202            }
9203
9204            if indexed_query_sheets.insert(region.sheet_id) {
9205                self.graph.prepare_sheet_index_for_query(region.sheet_id);
9206                let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
9207                    .saturating_mul(32);
9208                self.reserve_graph_source_scratch(bytes)?;
9209                discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
9210            }
9211            let spill_anchors = self.graph.spill_anchors_in_region(
9212                region.sheet_id,
9213                region.start_row - 1,
9214                region.start_col - 1,
9215                region.end_row - 1,
9216                region.end_col - 1,
9217            );
9218            for anchor in spill_anchors {
9219                self.target_preparation_checkpoint(options.deadline, 1)?;
9220                symbol_vertices.push_back(anchor);
9221            }
9222            let vertices = self.graph.vertices_in_region(
9223                region.sheet_id,
9224                region.start_row - 1,
9225                region.end_row - 1,
9226                region.start_col - 1,
9227                region.end_col - 1,
9228            );
9229            for vertex in vertices {
9230                self.target_preparation_checkpoint(options.deadline, 1)?;
9231                symbol_vertices.push_back(vertex);
9232            }
9233        }
9234
9235        #[cfg(test)]
9236        self.target_preparation_fault(
9237            crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
9238        )?;
9239
9240        if package_encountered {
9241            Self::widen_target_preparation(
9242                options.opaque_policy,
9243                &mut scope,
9244                &mut reasons,
9245                OpaqueReason::UnsupportedSourceSemantics,
9246            )?;
9247            self.target_preparation_checkpoint(options.deadline, 0)?;
9248            let selected_count = self.staged_formula_count();
9249            let selected_packages = self
9250                .staged_formulas
9251                .values()
9252                .filter_map(|staged| staged.deferred_package.as_ref())
9253                .map(|package| package.families.len() + package.partitioned_families.len())
9254                .sum();
9255            self.formula_parse_diagnostics.truncate(diagnostics_len);
9256            self.last_formula_ingest_report = report_len;
9257            #[cfg(test)]
9258            if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9259                hook();
9260            }
9261            self.target_preparation_checkpoint(options.deadline, 0)?;
9262            #[cfg(test)]
9263            self.target_preparation_fault(
9264                crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9265            )?;
9266            let current_revisions = self.preparation_revisions();
9267            if let Some(reason) = Self::preparation_revision_stale_reason(
9268                &assumptions,
9269                &current_revisions,
9270                &planning_requests,
9271                true,
9272            ) {
9273                return Err(Self::preparation_stale(
9274                    reason,
9275                    "target compatibility preparation plan is stale",
9276                ));
9277            }
9278            #[cfg(test)]
9279            self.target_preparation_fault(
9280                crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9281            )?;
9282            let commit_work_before = self
9283                .active_resource_ledger
9284                .as_ref()
9285                .map_or(0, |ledger| ledger.snapshot().work_charged);
9286            let commit_started = crate::instant::FzInstant::now();
9287            self.build_graph_all_unobserved()?;
9288            let commit_window = commit_started.elapsed();
9289            let ledger_after = self
9290                .active_resource_ledger
9291                .as_ref()
9292                .map(|ledger| ledger.snapshot());
9293            let actual_commit_work = ledger_after
9294                .map_or(0, |snapshot| snapshot.work_charged)
9295                .saturating_sub(commit_work_before);
9296            let observed_scratch_bytes = ledger_after
9297                .map_or(0, |snapshot| snapshot.scratch_peak)
9298                .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
9299            let revisions = assumptions.clone();
9300            let report = PreparedTargetGraphReport {
9301                request_id: request_id.unwrap_or_default(),
9302                requested_targets: targets.len(),
9303                normalized_regions: visited_regions.len(),
9304                normalized_target_list: normalized,
9305                selected_staged_cells: selected_count,
9306                selected_source_families: selected_packages,
9307                retained_staged_cells: self.staged_formula_count(),
9308                selected_cells,
9309                retained_cells: Vec::new(),
9310                widened_scope: PrepareScope::Workbook,
9311                widening_reasons: reasons,
9312                revisions,
9313                commit_window,
9314                estimated_scratch_bytes: discovery_scratch_reserved
9315                    .saturating_add((selected_count as u64).saturating_mul(256)),
9316                observed_scratch_bytes,
9317                estimated_commit_work: selected_count as u64,
9318                actual_commit_work,
9319                outcome: PreparationOutcome::CompatibilityPrepared,
9320            };
9321            self.observe_target_preparation_report(&report);
9322            return Ok(report);
9323        }
9324
9325        prepared.sort_by_key(|formula| formula.lease.insertion_order);
9326        for package in &prepared_packages {
9327            if let Some(points) = &package.selected_points {
9328                selected_cells.extend(points.iter().filter_map(|&(row, col)| {
9329                    formualizer_common::RangeAddress::new(&package.sheet, row, col, row, col).ok()
9330                }));
9331                continue;
9332            }
9333            selected_cells.extend(package.replay_records.iter().filter_map(|record| {
9334                formualizer_common::RangeAddress::new(
9335                    package.sheet.clone(),
9336                    record.row,
9337                    record.col,
9338                    record.row,
9339                    record.col,
9340                )
9341                .ok()
9342            }));
9343        }
9344        let (legacy_graph, planned_formula_count) =
9345            self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
9346        let new_vertices = legacy_graph.new_vertex_count();
9347        let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
9348            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9349        })?;
9350        let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
9351            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9352        })?;
9353        let current = self.graph.baseline_stats();
9354        let final_vertices = current
9355            .graph_vertex_count
9356            .checked_add(new_vertices)
9357            .ok_or_else(|| {
9358                crate::engine::ResourceLedgerError::Exhausted(
9359                    formualizer_common::ResourceExhaustionDetail {
9360                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9361                        limit: u64::MAX,
9362                        observed: u64::MAX,
9363                        request_id,
9364                    },
9365                )
9366                .into_excel_error()
9367            })?;
9368        let final_edges = current
9369            .graph_edge_count
9370            .checked_sub(removed_edges)
9371            .and_then(|count| count.checked_add(new_edges))
9372            .ok_or_else(|| {
9373                crate::engine::ResourceLedgerError::Exhausted(
9374                    formualizer_common::ResourceExhaustionDetail {
9375                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9376                        limit: u64::MAX,
9377                        observed: u64::MAX,
9378                        request_id,
9379                    },
9380                )
9381                .into_excel_error()
9382            })?;
9383        #[cfg(test)]
9384        self.target_preparation_fault(
9385            crate::engine::target_preparation::TargetPreparationFault::Admission,
9386        )?;
9387        let resource = |reason, limit: u64, observed: u64| {
9388            crate::engine::ResourceLedgerError::Exhausted(
9389                formualizer_common::ResourceExhaustionDetail {
9390                    reason,
9391                    limit,
9392                    observed,
9393                    request_id,
9394                },
9395            )
9396            .into_excel_error()
9397        };
9398        let admission = crate::engine::resource_ledger::GraphAdmission {
9399            final_vertices,
9400            final_edges,
9401            materialization_cells: planned_formula_count as u64,
9402            added_vertices: new_vertices,
9403            added_edges: new_edges,
9404        };
9405        let materialized_bytes = admission
9406            .materialized_graph_bytes()
9407            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
9408        if let Err(error) = self.preflight_graph_admission(admission) {
9409            if let formualizer_common::ExcelErrorExtra::Resource { detail } = &error.extra {
9410                self.observe_target_admission_failure(detail.reason);
9411            }
9412            return Err(error);
9413        }
9414        let selected_package_records = prepared_packages
9415            .iter()
9416            .map(|package| package.replay_records.len() as u64)
9417            .sum::<u64>();
9418        let planned_working_bytes = (prepared.len() as u64)
9419            .saturating_add(selected_package_records)
9420            .saturating_mul(256)
9421            .saturating_add((visited_regions.len() as u64).saturating_mul(128))
9422            .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
9423            .saturating_add((selected.len() as u64).saturating_mul(96))
9424            .saturating_add(materialized_bytes);
9425        let residual_scratch = selected_package_points
9426            .iter()
9427            .map(|(sheet, points)| {
9428                let source = self
9429                    .staged_formulas
9430                    .get(sheet)
9431                    .unwrap()
9432                    .deferred_package
9433                    .as_ref()
9434                    .unwrap();
9435                (source.families.len() as u64)
9436                    .saturating_mul(256)
9437                    .saturating_add(
9438                        source
9439                            .partitioned_families
9440                            .iter()
9441                            .map(|family| {
9442                                256u64
9443                                    .saturating_add(family.fragments.len() as u64 * 32)
9444                                    .saturating_add(family.legacy_members.len() as u64 * 32)
9445                            })
9446                            .sum::<u64>(),
9447                    )
9448                    // Direct complete domains need coordinate suppression, not
9449                    // a per-member residual split/proof or legacy AST reserve.
9450                    .saturating_add(points.len() as u64 * 32)
9451                    .saturating_add(
9452                        prepared_packages
9453                            .iter()
9454                            .filter(|package| &package.sheet == sheet)
9455                            .map(|package| package.replay_records.len() as u64 * 480)
9456                            .sum::<u64>(),
9457                    )
9458            })
9459            .sum::<u64>();
9460        let scratch_bytes = discovery_scratch_reserved
9461            .saturating_add(planned_working_bytes)
9462            .saturating_add(residual_scratch);
9463        let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
9464        if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
9465            self.observe_target_admission_failure(
9466                formualizer_common::ResourceExhaustionReason::ScratchMemory,
9467            );
9468            return Err(error);
9469        }
9470
9471        let mut residual_sources = BTreeMap::new();
9472        let mut residual_owners = BTreeMap::new();
9473        for (sheet, points) in &selected_package_points {
9474            let selected: BTreeMap<_, BTreeSet<_>> = prepared_packages
9475                .iter()
9476                .filter(|package| &package.sheet == sheet)
9477                .flat_map(|package| &package.replay_records)
9478                .filter_map(|record| {
9479                    record.partition_owner.or(record.family).map(|owner| {
9480                        (
9481                            owner,
9482                            crate::engine::SourceCoord {
9483                                row: record.row - 1,
9484                                col: record.col - 1,
9485                            },
9486                        )
9487                    })
9488                })
9489                .fold(BTreeMap::new(), |mut map, (owner, coord)| {
9490                    map.entry(owner).or_default().insert(coord);
9491                    map
9492                });
9493            let metadata_work = self
9494                .staged_formulas
9495                .get(sheet)
9496                .unwrap()
9497                .deferred_package
9498                .as_ref()
9499                .map_or(0, |source| {
9500                    source.families.len() as u64
9501                        + source
9502                            .partitioned_families
9503                            .iter()
9504                            .map(|family| {
9505                                1 + family.fragments.len() as u64 * family.fragments.len() as u64
9506                                    + family.legacy_members.len() as u64
9507                            })
9508                            .sum::<u64>()
9509                });
9510            let split_work = selected
9511                .values()
9512                .map(|points| points.len() as u64 * 128)
9513                .sum::<u64>();
9514            self.target_preparation_checkpoint(
9515                options.deadline,
9516                metadata_work
9517                    .saturating_add(split_work)
9518                    .saturating_add(points.len() as u64),
9519            )?;
9520            let source = self
9521                .staged_formulas
9522                .get(sheet)
9523                .unwrap()
9524                .deferred_package
9525                .as_ref()
9526                .unwrap();
9527            let complete: BTreeSet<_> = prepared_packages
9528                .iter()
9529                .filter(|package| &package.sheet == sheet)
9530                .flat_map(|package| package.complete_selections.iter().copied())
9531                .collect();
9532            let residual = source
9533                .residual_sources(&selected, &complete, &self.workbook_load_limits)
9534                .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
9535            residual_owners.insert(
9536                sheet.clone(),
9537                residual
9538                    .1
9539                    .iter()
9540                    .map(|family| family.source_id)
9541                    .collect::<BTreeSet<_>>(),
9542            );
9543            residual_sources.insert(sheet.clone(), residual);
9544        }
9545
9546        // Reserve residual suppression before the revision-validated commit window.
9547        // Hash-set insertion and point-index removal below cannot allocate.
9548        for (sheet, points) in &selected_package_points {
9549            let package = self
9550                .staged_formulas
9551                .get_mut(sheet)
9552                .unwrap()
9553                .deferred_package
9554                .as_mut()
9555                .unwrap();
9556            let owner_count = prepared_packages
9557                .iter()
9558                .filter(|p| &p.sheet == sheet)
9559                .flat_map(|p| &p.replay_records)
9560                .filter_map(|record| record.partition_owner.or(record.family))
9561                .filter(|owner| residual_owners[sheet].contains(owner))
9562                .count();
9563            package
9564                .consumed_members
9565                .try_reserve(owner_count)
9566                .map_err(|_| {
9567                    resource(
9568                        formualizer_common::ResourceExhaustionReason::ScratchMemory,
9569                        0,
9570                        owner_count as u64 * 24,
9571                    )
9572                })?;
9573            package.suppressed.try_reserve(points.len()).map_err(|_| {
9574                resource(
9575                    formualizer_common::ResourceExhaustionReason::ScratchMemory,
9576                    0,
9577                    points.len() as u64 * 16,
9578                )
9579            })?;
9580        }
9581
9582        let estimated_commit_duration = std::time::Duration::from_nanos(
9583            (new_vertices as u64)
9584                .saturating_add(new_edges as u64)
9585                .saturating_add(prepared.len() as u64)
9586                .saturating_add(selected_package_records)
9587                .max(1)
9588                .saturating_mul(100),
9589        );
9590        if options.deadline.is_some_and(|deadline| {
9591            std::time::Instant::now()
9592                .checked_add(estimated_commit_duration)
9593                .is_none_or(|finish| finish > deadline)
9594        }) {
9595            self.observe_target_admission_failure(
9596                formualizer_common::ResourceExhaustionReason::Deadline,
9597            );
9598            return Err(resource(
9599                formualizer_common::ResourceExhaustionReason::Deadline,
9600                0,
9601                1,
9602            ));
9603        }
9604        #[cfg(test)]
9605        if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9606            hook();
9607        }
9608        self.target_preparation_checkpoint(options.deadline, 0)?;
9609        #[cfg(test)]
9610        self.target_preparation_fault(
9611            crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9612        )?;
9613        let current_revisions = self.preparation_revisions();
9614        let staged_leases_match = prepared.iter().all(|formula| {
9615            self.staged_formula_index
9616                .lease_matches(&formula.sheet, formula.lease)
9617        }) && prepared_packages.iter().all(|package| {
9618            self.staged_formula_index
9619                .package_lease_matches(&package.sheet, package.lease)
9620        });
9621        let stale_reason = Self::preparation_revision_stale_reason(
9622            &assumptions,
9623            &current_revisions,
9624            &planning_requests,
9625            staged_leases_match,
9626        );
9627        if let Some(reason) = stale_reason {
9628            return Err(Self::preparation_stale(
9629                reason,
9630                "target graph preparation plan is stale",
9631            ));
9632        }
9633        #[cfg(test)]
9634        self.target_preparation_fault(
9635            crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9636        )?;
9637        self.graph
9638            .validate_prepared_legacy_graph_plan(&legacy_graph)
9639            .map_err(|error| {
9640                Self::preparation_stale(
9641                    formualizer_common::PreparationStaleReason::Graph,
9642                    format!("target graph preparation plan is stale: {error}"),
9643                )
9644            })?;
9645        #[cfg(test)]
9646        self.target_preparation_fault(
9647            crate::engine::target_preparation::TargetPreparationFault::Reservation,
9648        )?;
9649        self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
9650        self.formula_parse_diagnostics
9651            .try_reserve(pending_diagnostics.len())
9652            .map_err(|_| {
9653                resource(
9654                    formualizer_common::ResourceExhaustionReason::Admission,
9655                    pending_diagnostics.len() as u64,
9656                    pending_diagnostics.len() as u64,
9657                )
9658            })?;
9659        self.target_preparation_checkpoint(options.deadline, 0)?;
9660        #[cfg(test)]
9661        self.target_preparation_fault(
9662            crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
9663        )?;
9664
9665        let commit_started = crate::instant::FzInstant::now();
9666        let committed = self
9667            .graph
9668            .apply_prevalidated_legacy_graph_plan(legacy_graph);
9669        for formula in &prepared {
9670            let removed = self
9671                .staged_formulas
9672                .get_mut(&formula.sheet)
9673                .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
9674            debug_assert!(removed.is_some());
9675            let index_removed = self.staged_formula_index.remove(
9676                &formula.sheet,
9677                formula.lease.row,
9678                formula.lease.col,
9679            );
9680            debug_assert!(index_removed);
9681        }
9682        for package in &prepared_packages {
9683            if let Some(points) = &package.selected_points {
9684                let staged = self.staged_formulas.get_mut(&package.sheet).unwrap();
9685                let source = staged.deferred_package.as_mut().unwrap();
9686                source.consumed_engine = Some(Arc::clone(&self.source_formula_token));
9687                // The revision-validated graph commit establishes this exact source
9688                // ownership proof. Later edits retain the exclusion, not old text.
9689                source
9690                    .consumed_members
9691                    .extend(package.replay_records.iter().filter_map(|record| {
9692                        record
9693                            .partition_owner
9694                            .or(record.family)
9695                            .filter(|owner| residual_owners[&package.sheet].contains(owner))
9696                            .map(|owner| {
9697                                (
9698                                    owner,
9699                                    crate::engine::SourceCoord {
9700                                        row: record.row - 1,
9701                                        col: record.col - 1,
9702                                    },
9703                                )
9704                            })
9705                    }));
9706                source.suppressed.extend(points.iter().copied());
9707                source.source_accounted = true;
9708                self.staged_formula_index
9709                    .consume_package_points(&package.sheet, points);
9710                if source.suppressed.len() >= source.source_coordinates.len() {
9711                    staged.deferred_package = None;
9712                    self.staged_formula_index.set_package(&package.sheet, None);
9713                }
9714            } else {
9715                let removed = self
9716                    .staged_formulas
9717                    .get_mut(&package.sheet)
9718                    .and_then(|staged| staged.deferred_package.take());
9719                debug_assert!(removed.is_some());
9720                self.staged_formula_index.set_package(&package.sheet, None);
9721            }
9722        }
9723        for (sheet, (families, partitions)) in residual_sources {
9724            if let Some(source) = self
9725                .staged_formulas
9726                .get_mut(&sheet)
9727                .and_then(|staged| staged.deferred_package.as_mut())
9728            {
9729                source.families = families;
9730                source.partitioned_families = partitions;
9731                source
9732                    .consumed_members
9733                    .retain(|(owner, _)| residual_owners[&sheet].contains(owner));
9734                self.staged_formula_index.update_package_family_count(
9735                    &sheet,
9736                    source.families.len() + source.partitioned_families.len(),
9737                );
9738            }
9739        }
9740        let empty_sheets = self
9741            .staged_formulas
9742            .iter()
9743            .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
9744            .collect::<Vec<_>>();
9745        for sheet in empty_sheets {
9746            self.staged_formulas.remove(&sheet);
9747        }
9748        if committed > 0 {
9749            self.mark_topology_edited();
9750        }
9751        self.formula_parse_diagnostics.extend(pending_diagnostics);
9752        if !prepared.is_empty() || !prepared_packages.is_empty() {
9753            let mut ingest_delta = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
9754            ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
9755                prepared_packages
9756                    .iter()
9757                    .map(|package| {
9758                        (package.replay_records.len() as u64).saturating_add(
9759                            if package.selected_points.is_some() {
9760                                package.direct_cells
9761                            } else {
9762                                0
9763                            },
9764                        )
9765                    })
9766                    .sum::<u64>(),
9767            );
9768            ingest_delta.graph_formula_cells_materialized = committed as u64;
9769            ingest_delta.graph_vertices_created = new_vertices as u64;
9770            ingest_delta.graph_edges_created = new_edges as u64;
9771            for package in &prepared_packages {
9772                let source = &package.source_report;
9773                ingest_delta.source_formula_events = ingest_delta
9774                    .source_formula_events
9775                    .saturating_add(source.source_formula_events);
9776                ingest_delta.source_formula_records_spooled = ingest_delta
9777                    .source_formula_records_spooled
9778                    .saturating_add(source.source_formula_records_spooled);
9779                ingest_delta.source_spool_encoded_bytes = ingest_delta
9780                    .source_spool_encoded_bytes
9781                    .saturating_add(source.source_spool_encoded_bytes);
9782                ingest_delta.source_spool_peak_memory_bytes = ingest_delta
9783                    .source_spool_peak_memory_bytes
9784                    .max(source.source_spool_peak_memory_bytes);
9785                ingest_delta.source_spool_spilled_bytes = ingest_delta
9786                    .source_spool_spilled_bytes
9787                    .saturating_add(source.source_spool_spilled_bytes);
9788                ingest_delta.source_spool_spill_files = ingest_delta
9789                    .source_spool_spill_files
9790                    .saturating_add(source.source_spool_spill_files);
9791                ingest_delta.source_spool_replays = ingest_delta
9792                    .source_spool_replays
9793                    .saturating_add(source.source_spool_replays)
9794                    .saturating_add(package.spool_replays);
9795                ingest_delta.source_families_seen = ingest_delta
9796                    .source_families_seen
9797                    .saturating_add(source.families_seen);
9798                ingest_delta.source_family_cells_seen = ingest_delta
9799                    .source_family_cells_seen
9800                    .saturating_add(source.family_cells_seen);
9801                ingest_delta.source_family_shadow_eligible = ingest_delta
9802                    .source_family_shadow_eligible
9803                    .saturating_add(source.source_clean_families);
9804                ingest_delta.source_family_shadow_eligible_cells = ingest_delta
9805                    .source_family_shadow_eligible_cells
9806                    .saturating_add(source.source_clean_cells);
9807                ingest_delta.source_partitioned_families_seen = ingest_delta
9808                    .source_partitioned_families_seen
9809                    .saturating_add(source.source_fragmentable_families);
9810                ingest_delta.source_partition_holes = ingest_delta
9811                    .source_partition_holes
9812                    .saturating_add(source.source_hole_exclusions);
9813                ingest_delta.source_partition_ordinary_exceptions = ingest_delta
9814                    .source_partition_ordinary_exceptions
9815                    .saturating_add(source.source_ordinary_exclusions);
9816                ingest_delta.source_partition_surviving_cells = ingest_delta
9817                    .source_partition_surviving_cells
9818                    .saturating_add(source.source_fragmentable_cells);
9819                for (reason, count) in &source.fallback_reasons {
9820                    let total = ingest_delta
9821                        .fallback_reasons
9822                        .entry(reason.clone())
9823                        .or_default();
9824                    *total = total.saturating_add(*count);
9825                }
9826
9827                ingest_delta.source_family_fallback = ingest_delta
9828                    .source_family_fallback
9829                    .saturating_add(source.families_seen);
9830                ingest_delta.source_family_fallback_cells = ingest_delta
9831                    .source_family_fallback_cells
9832                    .saturating_add(source.family_cells_seen);
9833            }
9834            self.record_formula_ingest_report(ingest_delta);
9835        }
9836        let commit_window = commit_started.elapsed();
9837        let retained_cells = self
9838            .staged_formula_index
9839            .all_leases()
9840            .into_iter()
9841            .filter_map(|(sheet, lease)| {
9842                formualizer_common::RangeAddress::new(
9843                    sheet, lease.row, lease.col, lease.row, lease.col,
9844                )
9845                .ok()
9846            })
9847            .collect::<Vec<_>>();
9848        let observed_scratch_bytes = self
9849            .active_resource_ledger
9850            .as_ref()
9851            .map_or(0, |ledger| ledger.snapshot().scratch_peak)
9852            .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
9853        let committed_spans = 0u64;
9854        let selected_source_families = prepared_packages
9855            .iter()
9856            .filter(|package| package.selected_points.is_none())
9857            .map(|package| package.lease.family_count)
9858            .sum::<usize>()
9859            + prepared_packages
9860                .iter()
9861                .filter(|package| package.selected_points.is_some())
9862                .flat_map(|package| package.replay_records.iter())
9863                .filter_map(|record| record.partition_owner.or(record.family))
9864                .chain(
9865                    prepared_packages
9866                        .iter()
9867                        .flat_map(|package| package.complete_selections.iter().copied()),
9868                )
9869                .collect::<BTreeSet<_>>()
9870                .len();
9871        let selected_staged_cells = prepared.len().saturating_add(
9872            prepared_packages
9873                .iter()
9874                .map(|package| {
9875                    package
9876                        .selected_points
9877                        .as_ref()
9878                        .map_or(package.replay_records.len(), BTreeSet::len)
9879                })
9880                .sum::<usize>(),
9881        );
9882        let actual_commit_work = (new_vertices as u64)
9883            .saturating_add(new_edges as u64)
9884            .saturating_add(committed as u64)
9885            .saturating_add(committed_spans)
9886            .saturating_add(prepared.len() as u64)
9887            .saturating_add(
9888                prepared_packages
9889                    .iter()
9890                    .map(|package| {
9891                        package
9892                            .selected_points
9893                            .as_ref()
9894                            .map_or(1, |points| points.len() as u64)
9895                    })
9896                    .sum::<u64>(),
9897            );
9898        let report = PreparedTargetGraphReport {
9899            request_id: request_id.unwrap_or_default(),
9900            requested_targets: targets.len(),
9901            normalized_regions: visited_regions.len(),
9902            normalized_target_list: normalized,
9903            selected_staged_cells,
9904            selected_source_families,
9905            retained_staged_cells: self.staged_formula_count(),
9906            selected_cells,
9907            retained_cells,
9908            widened_scope: scope,
9909            widening_reasons: reasons,
9910            revisions: assumptions,
9911            commit_window,
9912            estimated_scratch_bytes: scratch_bytes,
9913            observed_scratch_bytes,
9914            estimated_commit_work: (new_vertices as u64)
9915                .saturating_add(new_edges as u64)
9916                .saturating_add(prepared.len() as u64)
9917                .saturating_add(selected_package_records),
9918            actual_commit_work,
9919            outcome: PreparationOutcome::Prepared,
9920        };
9921        self.observe_target_preparation_report(&report);
9922        Ok(report)
9923    }
9924
9925    /// Build graph for all staged formulas.
9926    pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
9927        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
9928            engine.build_graph_all_unobserved()
9929        })
9930    }
9931
9932    fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
9933        let selected = self.staged_formula_count();
9934        let started = crate::instant::FzInstant::now();
9935        self.resource_checkpoint(selected as u64)?;
9936        let scratch_bytes = (selected as u64).saturating_mul(256);
9937        let result = self.with_request_scratch(scratch_bytes, |engine| {
9938            let index_snapshot = engine.staged_formula_index.clone();
9939            let collected = std::mem::take(&mut engine.staged_formulas)
9940                .into_iter()
9941                .collect();
9942            engine.staged_formula_index.clear_all();
9943            engine.build_graph_from_staged_batches(collected, false, index_snapshot)
9944        });
9945        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
9946        result
9947    }
9948
9949    /// Build graph for specific sheets (consuming only those staged entries).
9950    pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
9951        &mut self,
9952        sheets: I,
9953    ) -> Result<(), formualizer_parse::ExcelError> {
9954        let mut sheets = sheets.into_iter();
9955        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
9956            let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
9957            engine.with_request_scratch(name_scratch, |engine| {
9958                // Allocation failure follows the baseline process-fatal policy; it is not a
9959                // recoverable resource error or a new staged-preparation route.
9960                let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
9961                engine.charge_bounded_work(names.len() as u64)?;
9962                engine.build_graph_for_sheet_names_unobserved(names)
9963            })
9964        })
9965    }
9966
9967    fn build_graph_for_sheet_names_unobserved(
9968        &mut self,
9969        sheets: Vec<String>,
9970    ) -> Result<(), formualizer_parse::ExcelError> {
9971        let started = crate::instant::FzInstant::now();
9972        let selected = sheets
9973            .iter()
9974            .filter_map(|sheet| self.staged_formulas.get(sheet))
9975            .map(StagedSheet::len)
9976            .sum::<usize>();
9977        self.resource_checkpoint(selected as u64)?;
9978        let scratch_bytes = (selected as u64).saturating_mul(256);
9979        self.reserve_request_scratch(scratch_bytes)?;
9980        let index_snapshot = self.staged_formula_index.clone();
9981        let mut collected = Vec::new();
9982        for sheet in sheets {
9983            if let Some(staged) = self.staged_formulas.remove(&sheet) {
9984                self.index_removed_staged_sheet(&sheet, &staged);
9985                collected.push((sheet, staged));
9986            }
9987        }
9988        let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
9989        self.release_request_scratch(scratch_bytes);
9990        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
9991        result
9992    }
9993
9994    fn build_graph_from_staged_batches(
9995        &mut self,
9996        collected: StagedFormulaBatches,
9997        share_parse_cache_across_sheets: bool,
9998        staged_index_snapshot: StagedFormulaIndex,
9999    ) -> Result<(), formualizer_parse::ExcelError> {
10000        if collected.is_empty() {
10001            return Ok(());
10002        }
10003        for (sheet, _) in &collected {
10004            let _ = self.add_sheet(sheet);
10005        }
10006
10007        let diagnostics_len = self.formula_parse_diagnostics.len();
10008        let mut collected = collected;
10009        let prepared = match self
10010            .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
10011        {
10012            Ok(prepared) => prepared,
10013            Err(error) => {
10014                self.formula_parse_diagnostics.truncate(diagnostics_len);
10015                for (sheet, staged) in collected {
10016                    self.restore_staged_sheet(sheet, staged);
10017                }
10018                self.staged_formula_index = staged_index_snapshot;
10019                return Err(error);
10020            }
10021        };
10022        let (ordinary, compressed, direct, may_fail) = prepared;
10023
10024        // A first build (no formula in the graph yet) goes through the eager
10025        // first load's machinery: the builder pre-allocates each column's
10026        // targets as one id run, installs family members as virtual runs and
10027        // builds the authority once at the end (decision 26).
10028        // The first load's builder plans and applies one chunk at a time, so
10029        // a planning error would leave earlier chunks in the graph; the
10030        // incremental path plans everything first. Check every distinct
10031        // formula first, in the incremental path's order (it reports the
10032        // same first error and leaves the graph untouched).
10033        let first_build = if self.deferred_build_can_be_first_load() {
10034            if let Err(error) = self.check_staged_formula_plans(
10035                ordinary
10036                    .iter()
10037                    .chain(compressed.iter().map(|(batch, _)| batch)),
10038                &may_fail,
10039            ) {
10040                self.formula_parse_diagnostics.truncate(diagnostics_len);
10041                for (sheet, staged) in collected {
10042                    self.restore_staged_sheet(sheet, staged);
10043                }
10044                self.staged_formula_index = staged_index_snapshot;
10045                return Err(error);
10046            }
10047            self.deferred_build_as_first_load()
10048        } else {
10049            None
10050        };
10051        let built_sheets: Vec<String> = if first_build.is_some() {
10052            collected.iter().map(|(sheet, _)| sheet.clone()).collect()
10053        } else {
10054            Vec::new()
10055        };
10056
10057        // Keep the original source/spool alive through every fallible ingestion route.
10058        // Graph admission may have committed a prefix; replay replaces those placements
10059        // rather than treating their cached values as authoritative source.
10060        let result = (|| {
10061            if !ordinary.is_empty() {
10062                self.ingest_formula_batches(ordinary)?;
10063            }
10064            if !compressed.is_empty() {
10065                self.ingest_compressed_formula_source_batches(compressed)?;
10066            }
10067            if !direct.is_empty() {
10068                self.finish_compressed_formula_sources(direct)?;
10069            }
10070            Ok(())
10071        })();
10072        if let Some(saved) = first_build {
10073            self.leave_deferred_first_load(saved, &built_sheets);
10074        }
10075        if let Err(error) = result {
10076            self.formula_parse_diagnostics.truncate(diagnostics_len);
10077            for (sheet, staged) in collected {
10078                self.restore_staged_sheet(sheet, staged);
10079            }
10080            self.staged_formula_index = staged_index_snapshot;
10081            return Err(error);
10082        }
10083        self.dedup_formula_parse_diagnostics_since(diagnostics_len);
10084        Ok(())
10085    }
10086
10087    /// Enter the first-load ingest mode for a deferred build when the graph
10088    /// holds no formula yet and no load is in progress: the settings the
10089    /// calamine loader uses for its eager first load (lazy sheet index, no
10090    /// small-range expansion, first-load fast path). Returns the settings
10091    /// to restore, or `None` when the build takes the incremental path.
10092    fn deferred_build_can_be_first_load(&self) -> bool {
10093        !self.graph.first_load_assume_new()
10094            && !self.graph_admission_enabled()
10095            && self.graph.formula_vertex_count() == 0
10096    }
10097
10098    /// Plan each distinct formula of `batches` that may fail planning once
10099    /// (a member is planned through its template, staged before it) and
10100    /// drop the plans: the first planning error, in the order the
10101    /// incremental ingest meets it.
10102    fn check_staged_formula_plans<'b>(
10103        &mut self,
10104        batches: impl Iterator<Item = &'b FormulaIngestBatch>,
10105        may_fail: &FxHashSet<crate::engine::arena::AstNodeId>,
10106    ) -> Result<(), ExcelError> {
10107        if may_fail.is_empty() {
10108            return Ok(());
10109        }
10110        let mut seen: FxHashSet<(SheetId, crate::engine::arena::AstNodeId)> = FxHashSet::default();
10111        for batch in batches {
10112            let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
10113                ExcelError::new(ExcelErrorKind::Ref)
10114                    .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
10115            })?;
10116            let mut pipeline = self.ingest_pipeline();
10117            for record in &batch.formulas {
10118                if record.member_anchor.is_some()
10119                    || !may_fail.contains(&record.ast_id)
10120                    || !seen.insert((sheet_id, record.ast_id))
10121                {
10122                    continue;
10123                }
10124                let placement = CellRef::new(
10125                    sheet_id,
10126                    Coord::from_excel(record.row, record.col, true, true),
10127                );
10128                pipeline.ingest_formula(
10129                    FormulaAstInput::RawArena(record.ast_id),
10130                    placement,
10131                    None,
10132                )?;
10133            }
10134        }
10135        Ok(())
10136    }
10137
10138    fn deferred_build_as_first_load(&mut self) -> Option<(crate::engine::SheetIndexMode, usize)> {
10139        if !self.deferred_build_can_be_first_load() {
10140            return None;
10141        }
10142        let saved = (
10143            self.graph.get_config().sheet_index_mode,
10144            self.config.range_expansion_limit,
10145        );
10146        self.graph
10147            .set_sheet_index_mode(crate::engine::SheetIndexMode::Lazy);
10148        self.config.range_expansion_limit = 0;
10149        self.graph.set_first_load_assume_new(true);
10150        self.graph.reset_ensure_touched();
10151        Some(saved)
10152    }
10153
10154    /// Leave the first-load mode of [`Self::deferred_build_as_first_load`]:
10155    /// the authority is built once here, as at the end of an eager load.
10156    fn leave_deferred_first_load(
10157        &mut self,
10158        (index_mode, range_limit): (crate::engine::SheetIndexMode, usize),
10159        sheets: &[String],
10160    ) {
10161        self.graph.set_first_load_assume_new(false);
10162        self.graph.reset_ensure_touched();
10163        self.graph.set_sheet_index_mode(index_mode);
10164        self.config.range_expansion_limit = range_limit;
10165        for sheet in sheets {
10166            self.graph.finalize_sheet_index(sheet);
10167        }
10168    }
10169
10170    fn prepare_staged_formula_batches(
10171        &mut self,
10172        collected: &mut StagedFormulaBatches,
10173        share_parse_cache_across_sheets: bool,
10174    ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
10175        let mut ordinary = Vec::new();
10176        let mut compressed = Vec::new();
10177        let mut direct = Vec::new();
10178        let mut may_fail = FxHashSet::default();
10179        let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
10180            rustc_hash::FxHashMap::default();
10181        cache.reserve(4096);
10182
10183        for (sheet, staged) in collected {
10184            if !share_parse_cache_across_sheets {
10185                cache.clear();
10186            }
10187            // Load-time family grouping, as the eager first load does:
10188            // relative copies of the formula above (or to the left) become
10189            // members of its family and are never interned.
10190            let mut grouper = crate::engine::FormulaFamilyGrouper::new();
10191            // The staged texts, then the deferred package's replayed ones,
10192            // are walked once (no combined copy: a replay can be the whole
10193            // sheet).
10194            let mut replayed_records = Vec::new();
10195            let deferred_source = None;
10196            let mut deferred_fallback = None;
10197            if let Some(package) = staged.deferred_package.as_mut() {
10198                if package.sheet_name != *sheet {
10199                    return Err(ExcelError::new(ExcelErrorKind::Value)
10200                        .with_message("deferred formula package sheet mismatch"));
10201                }
10202                let eligible: Vec<_> = package
10203                    .families
10204                    .iter()
10205                    .filter(|family| !package.invalidated.contains(&family.source_id))
10206                    .cloned()
10207                    .collect();
10208                let eligible_partitions: Vec<_> = package
10209                    .partitioned_families
10210                    .iter()
10211                    .filter(|family| !package.invalidated.contains(&family.source_id))
10212                    .cloned()
10213                    .collect();
10214
10215                let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
10216                for partition in &eligible_partitions {
10217                    replay_disposition
10218                        .register_partition(partition, false)
10219                        .map_err(|reason| {
10220                            ExcelError::new(ExcelErrorKind::Value).with_message(reason)
10221                        })?;
10222                }
10223                replay_disposition
10224                    .extend_suppressed_excel_coords(package.suppressed.iter().copied());
10225                let replayed = package
10226                    .replay
10227                    .lock()
10228                    .map_err(|_| {
10229                        ExcelError::new(ExcelErrorKind::Value)
10230                            .with_message("deferred formula spool lock poisoned")
10231                    })?
10232                    .replay(&replay_disposition)
10233                    .map_err(|message| {
10234                        ExcelError::new(ExcelErrorKind::Value).with_message(message)
10235                    })?;
10236                replayed_records = replayed;
10237                let mut report = package.accounting_report();
10238                report.source_spool_replays = report.source_spool_replays.saturating_add(1);
10239                deferred_fallback = Some((report, package.families.clone(), eligible_partitions));
10240            }
10241
10242            let n_entries = staged.entries.len() + replayed_records.len();
10243            let entries = staged
10244                .entries
10245                .iter()
10246                .cloned()
10247                .map(|(row, col, text)| (row, col, text, None))
10248                .chain(replayed_records.into_iter().map(|record| {
10249                    (
10250                        record.row,
10251                        record.col,
10252                        record.text,
10253                        Some((record.source_order, record.family, record.partition_owner)),
10254                    )
10255                }));
10256            let mut formulas = Vec::with_capacity(n_entries);
10257            let staged_order_base = u64::MAX.saturating_sub(n_entries as u64);
10258            for (entry_index, (row, col, txt, source_proof)) in entries.enumerate() {
10259                let key = if txt.starts_with('=') {
10260                    txt
10261                } else {
10262                    format!("={txt}")
10263                };
10264                let staged_record = if let Some(cached) = cache.get(&key) {
10265                    cached.map(|ast_id| {
10266                        self.note_staged_formula(&mut grouper, row, col, ast_id);
10267                        FormulaIngestRecord::new(row, col, ast_id, Some(Arc::<str>::from(key)))
10268                    })
10269                } else {
10270                    let parsed = match formualizer_parse::parser::parse(&key) {
10271                        Ok(parsed) => Some(parsed),
10272                        Err(error) => self.handle_formula_parse_error(
10273                            sheet,
10274                            row,
10275                            col,
10276                            &key,
10277                            error.to_string(),
10278                        )?,
10279                    };
10280                    match parsed {
10281                        Some(ast) => {
10282                            let record = self.stage_formula_ast(&mut grouper, row, col, &ast, None);
10283                            if !record.is_family_member() && self.formula_may_fail_planning(&ast) {
10284                                may_fail.insert(record.ast_id);
10285                            }
10286                            // A member's text is not worth caching: relative
10287                            // copies do not repeat their text.
10288                            if record.is_family_member() {
10289                                Some(record)
10290                            } else {
10291                                let ast_id = record.ast_id;
10292                                cache.insert(key.clone(), Some(ast_id));
10293                                Some(FormulaIngestRecord::new(
10294                                    row,
10295                                    col,
10296                                    ast_id,
10297                                    Some(Arc::<str>::from(key)),
10298                                ))
10299                            }
10300                        }
10301                        None => {
10302                            cache.insert(key, None);
10303                            None
10304                        }
10305                    }
10306                };
10307
10308                if let Some(mut formula) = staged_record {
10309                    if let Some((order, family, owner)) = source_proof {
10310                        formula = formula.with_source_proof(order, family, owner);
10311                    } else if deferred_source.is_some() {
10312                        formula = formula.with_source_proof(
10313                            crate::engine::SourceFormulaOrder::new(
10314                                staged_order_base.saturating_add(entry_index as u64),
10315                            ),
10316                            None,
10317                            None,
10318                        );
10319                    }
10320                    formulas.push(formula);
10321                }
10322            }
10323
10324            let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
10325            if let Some((report, preparation)) = deferred_source {
10326                direct.push((batch, report, preparation));
10327            } else if let Some((report, families, partitions)) = deferred_fallback {
10328                let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
10329                    batch.sheet_name.clone(),
10330                    report,
10331                    families,
10332                    partitions,
10333                );
10334                compressed.push((batch, source_batch));
10335            } else if !batch.is_empty() {
10336                ordinary.push(batch);
10337            }
10338        }
10339        Ok((ordinary, compressed, direct, may_fail))
10340    }
10341
10342    /// Whether planning `ast` can fail: a reference to a sheet that does not
10343    /// exist, an external, 3-D or table reference, or a reversed range.
10344    /// Unqualified cell and range references, names, literals and calls
10345    /// always plan.
10346    fn formula_may_fail_planning(&self, ast: &formualizer_parse::parser::ASTNode) -> bool {
10347        use formualizer_parse::parser::{ASTNodeType, ReferenceType};
10348        let sheet_missing = |sheet: &Option<String>| {
10349            sheet
10350                .as_deref()
10351                .is_some_and(|s| self.graph.sheet_id(s).is_none())
10352        };
10353        match &ast.node_type {
10354            ASTNodeType::Literal(_) | ASTNodeType::Omitted => false,
10355            ASTNodeType::Reference { reference, .. } => match reference {
10356                ReferenceType::Cell { sheet, .. } => sheet_missing(sheet),
10357                ReferenceType::Range {
10358                    sheet,
10359                    start_row,
10360                    start_col,
10361                    end_row,
10362                    end_col,
10363                    ..
10364                } => {
10365                    sheet_missing(sheet)
10366                        || matches!((start_row, end_row), (Some(a), Some(b)) if a > b)
10367                        || matches!((start_col, end_col), (Some(a), Some(b)) if a > b)
10368                }
10369                ReferenceType::NamedRange(_) => false,
10370                _ => true,
10371            },
10372            ASTNodeType::UnaryOp { expr, .. } => self.formula_may_fail_planning(expr),
10373            ASTNodeType::BinaryOp { left, right, .. } => {
10374                self.formula_may_fail_planning(left) || self.formula_may_fail_planning(right)
10375            }
10376            ASTNodeType::Function { args, .. } => {
10377                args.iter().any(|a| self.formula_may_fail_planning(a))
10378            }
10379            ASTNodeType::Call { callee, args } => {
10380                self.formula_may_fail_planning(callee)
10381                    || args.iter().any(|a| self.formula_may_fail_planning(a))
10382            }
10383            ASTNodeType::Array(rows) => rows
10384                .iter()
10385                .flatten()
10386                .any(|a| self.formula_may_fail_planning(a)),
10387        }
10388    }
10389
10390    /// Begin bulk Arrow ingest for base values (Phase A)
10391    pub fn begin_bulk_ingest_arrow(
10392        &mut self,
10393    ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
10394        crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
10395    }
10396
10397    /// Begin bulk updates to Arrow store (Phase C)
10398    pub fn begin_bulk_update_arrow(
10399        &mut self,
10400    ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
10401        crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
10402    }
10403
10404    fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
10405        self.graph.sheet_id(sheet).ok_or(
10406            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
10407                name: sheet.to_string(),
10408                reason: "Unknown sheet".to_string(),
10409            },
10410        )
10411    }
10412
10413    fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
10414        if row_1based == 0 {
10415            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10416        }
10417        Ok(row_1based - 1)
10418    }
10419
10420    fn normalize_row_range_1based(
10421        start_row_1based: u32,
10422        end_row_1based: u32,
10423    ) -> Result<(u32, u32), crate::engine::EditorError> {
10424        if start_row_1based == 0 || end_row_1based == 0 {
10425            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10426        }
10427        if start_row_1based > end_row_1based {
10428            return Err(crate::engine::EditorError::TransactionFailed {
10429                reason: "Row range start is greater than end".to_string(),
10430            });
10431        }
10432        Ok((start_row_1based - 1, end_row_1based - 1))
10433    }
10434
10435    fn invalidate_row_visibility_mask_cache(&self) {
10436        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
10437            cache.clear();
10438        }
10439    }
10440
10441    /// Hidden rows changed: drop the cached masks and dirty the formulas
10442    /// that read row visibility (SUBTOTAL/AGGREGATE).
10443    fn row_visibility_changed(&mut self) {
10444        self.invalidate_row_visibility_mask_cache();
10445        self.graph.mark_row_visibility_readers_dirty();
10446    }
10447
10448    fn set_row_hidden_by_sheet_id(
10449        &mut self,
10450        sheet_id: SheetId,
10451        row0: u32,
10452        hidden: bool,
10453        source: RowVisibilitySource,
10454    ) -> bool {
10455        let changed = {
10456            let state = self.row_visibility.entry(sheet_id).or_default();
10457            state.set_row_hidden(row0, hidden, source)
10458        };
10459
10460        let remove_entry = self
10461            .row_visibility
10462            .get(&sheet_id)
10463            .map(|state| state.is_empty())
10464            .unwrap_or(false);
10465        if remove_entry {
10466            self.row_visibility.remove(&sheet_id);
10467        }
10468
10469        if changed {
10470            self.row_visibility_changed();
10471        }
10472
10473        changed
10474    }
10475
10476    fn set_rows_hidden_by_sheet_id(
10477        &mut self,
10478        sheet_id: SheetId,
10479        start_row0: u32,
10480        end_row0: u32,
10481        hidden: bool,
10482        source: RowVisibilitySource,
10483    ) -> bool {
10484        let changed = {
10485            let state = self.row_visibility.entry(sheet_id).or_default();
10486            state.set_rows_hidden(start_row0, end_row0, hidden, source)
10487        };
10488
10489        let remove_entry = self
10490            .row_visibility
10491            .get(&sheet_id)
10492            .map(|state| state.is_empty())
10493            .unwrap_or(false);
10494        if remove_entry {
10495            self.row_visibility.remove(&sheet_id);
10496        }
10497
10498        if changed {
10499            self.row_visibility_changed();
10500        }
10501
10502        changed
10503    }
10504
10505    fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
10506        if count == 0 {
10507            return;
10508        }
10509        let mut changed = false;
10510        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10511            changed = state.insert_rows(before0, count);
10512            state.is_empty()
10513        } else {
10514            false
10515        };
10516        if remove_entry {
10517            self.row_visibility.remove(&sheet_id);
10518        }
10519        if changed {
10520            self.row_visibility_changed();
10521        }
10522    }
10523
10524    fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
10525        if count == 0 {
10526            return;
10527        }
10528        let mut changed = false;
10529        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10530            changed = state.delete_rows(start0, count);
10531            state.is_empty()
10532        } else {
10533            false
10534        };
10535        if remove_entry {
10536            self.row_visibility.remove(&sheet_id);
10537        }
10538        if changed {
10539            self.row_visibility_changed();
10540        }
10541    }
10542
10543    fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10544        if let crate::engine::ChangeEvent::SetRowVisibility {
10545            sheet_id,
10546            row0,
10547            source,
10548            old_hidden,
10549            ..
10550        } = event
10551        {
10552            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
10553        }
10554    }
10555
10556    fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10557        if let crate::engine::ChangeEvent::SetRowVisibility {
10558            sheet_id,
10559            row0,
10560            source,
10561            new_hidden,
10562            ..
10563        } = event
10564        {
10565            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
10566        }
10567    }
10568
10569    fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10570        for event in events.iter().rev() {
10571            self.apply_inverse_row_visibility_event(event);
10572        }
10573    }
10574
10575    fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10576        for event in events {
10577            self.apply_forward_row_visibility_event(event);
10578        }
10579    }
10580
10581    fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10582        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10583            sheet,
10584            row,
10585            col,
10586            old,
10587            ..
10588        } = event
10589        {
10590            self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
10591        }
10592    }
10593
10594    fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10595        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10596            sheet,
10597            row,
10598            col,
10599            new,
10600            ..
10601        } = event
10602        {
10603            self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
10604        }
10605    }
10606
10607    /// Set a single cell's staged formula text to `target` (clearing it when
10608    /// `None`). Used by undo/redo replay of per-cell staged-formula deltas.
10609    fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
10610        match target {
10611            Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
10612            None => {
10613                self.clear_staged_formula_text(sheet, row, col);
10614            }
10615        }
10616    }
10617
10618    pub fn set_row_hidden(
10619        &mut self,
10620        sheet: &str,
10621        row_1based: u32,
10622        hidden: bool,
10623        source: RowVisibilitySource,
10624    ) -> Result<(), crate::engine::EditorError> {
10625        self.observe_function_semantic_epoch()
10626            .map_err(crate::engine::EditorError::Excel)?;
10627        self.observe_function_semantic_epoch()
10628            .map_err(crate::engine::EditorError::Excel)?;
10629        let sheet_id = self.ensure_known_sheet_id(sheet)?;
10630        let row0 = Self::normalize_row_1based(row_1based)?;
10631        if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
10632            self.record_structural_change(StructuralScope::Region(Region::whole_row(
10633                sheet_id, row0,
10634            )));
10635            self.mark_data_edited();
10636        }
10637        Ok(())
10638    }
10639
10640    pub fn set_rows_hidden(
10641        &mut self,
10642        sheet: &str,
10643        start_row_1based: u32,
10644        end_row_1based: u32,
10645        hidden: bool,
10646        source: RowVisibilitySource,
10647    ) -> Result<(), crate::engine::EditorError> {
10648        let sheet_id = self.ensure_known_sheet_id(sheet)?;
10649        let (start_row0, end_row0) =
10650            Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
10651        if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
10652            if start_row0 == end_row0 {
10653                self.record_structural_change(StructuralScope::Region(Region::whole_row(
10654                    sheet_id, start_row0,
10655                )));
10656            } else {
10657                self.record_structural_change(StructuralScope::Sheet(sheet_id));
10658            }
10659            self.mark_data_edited();
10660        }
10661        Ok(())
10662    }
10663
10664    pub fn is_row_hidden(
10665        &self,
10666        sheet: &str,
10667        row_1based: u32,
10668        source: Option<RowVisibilitySource>,
10669    ) -> Option<bool> {
10670        let sheet_id = self.graph.sheet_id(sheet)?;
10671        let row0 = row_1based.checked_sub(1)?;
10672        Some(
10673            self.row_visibility
10674                .get(&sheet_id)
10675                .map(|state| state.is_row_hidden(row0, source))
10676                .unwrap_or(false),
10677        )
10678    }
10679
10680    pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
10681        let sheet_id = self.graph.sheet_id(sheet)?;
10682        Some(
10683            self.row_visibility
10684                .get(&sheet_id)
10685                .map(|state| state.version())
10686                .unwrap_or(0),
10687        )
10688    }
10689
10690    /// Cells of a sheet-backed `view` whose formula (own or a family
10691    /// template) calls SUBTOTAL, or AGGREGATE when `include_aggregate`, as
10692    /// sorted, disjoint `(col, first_row, last_row)` offset intervals within
10693    /// the view. Each formula's call bits are precomputed in the AST arena;
10694    /// a filled-down family is checked once per run, not once per cell.
10695    fn nested_subtotal_cells_for_view(
10696        &self,
10697        view: &RangeView<'_>,
10698        include_aggregate: bool,
10699    ) -> Option<Vec<(usize, usize, usize)>> {
10700        if !view.is_sheet_backed() {
10701            return None;
10702        }
10703        let (rows, cols) = view.dims();
10704        if rows == 0 || cols == 0 {
10705            return Some(Vec::new());
10706        }
10707        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10708        let (r0, c0) = (view.start_row(), view.start_col());
10709        let mask = if include_aggregate {
10710            crate::engine::arena::SUBTOTAL_CALL | crate::engine::arena::AGGREGATE_CALL
10711        } else {
10712            crate::engine::arena::SUBTOTAL_CALL
10713        };
10714        let ds = self.graph.data_store();
10715        let mut found: Vec<(usize, usize, usize)> = self
10716            .graph
10717            .formula_intervals_in_region(
10718                sheet_id,
10719                r0 as u32,
10720                (r0 + rows - 1) as u32,
10721                c0 as u32,
10722                (c0 + cols - 1) as u32,
10723                |template| ds.ast_subtotal_calls(template) & mask != 0,
10724            )
10725            .into_iter()
10726            .map(|(col, lo, hi)| (col as usize - c0, lo as usize - r0, hi as usize - r0))
10727            .collect();
10728        found.sort_unstable();
10729        // Merge touching or overlapping intervals of a column so a binary
10730        // search finds the one covering a cell.
10731        let mut out: Vec<(usize, usize, usize)> = Vec::with_capacity(found.len());
10732        for (col, lo, hi) in found {
10733            match out.last_mut() {
10734                Some(last) if last.0 == col && lo <= last.2 + 1 => last.2 = last.2.max(hi),
10735                _ => out.push((col, lo, hi)),
10736            }
10737        }
10738        Some(out)
10739    }
10740
10741    fn build_row_visibility_mask_for_view(
10742        &self,
10743        view: &RangeView<'_>,
10744        mode: VisibilityMaskMode,
10745    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
10746        let sheet_rows = view.sheet().nrows as usize;
10747        if sheet_rows == 0 || view.start_row() >= sheet_rows {
10748            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
10749        }
10750
10751        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10752        let start_row0 = view.start_row() as u32;
10753        let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
10754        let version = self
10755            .row_visibility
10756            .get(&sheet_id)
10757            .map(|state| state.version())
10758            .unwrap_or(0);
10759        let key = VisibilityMaskCacheKey {
10760            sheet_id,
10761            start_row0,
10762            end_row0,
10763            mode,
10764            version,
10765        };
10766
10767        if let Ok(cache) = self.row_visibility_mask_cache.read()
10768            && let Some(mask) = cache.get(&key)
10769        {
10770            #[cfg(test)]
10771            visibility_mask_test_hooks::inc_hit();
10772            return Some(mask.clone());
10773        }
10774
10775        #[cfg(test)]
10776        visibility_mask_test_hooks::inc_miss();
10777
10778        let state = self.row_visibility.get(&sheet_id);
10779        let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
10780        for row0 in start_row0..=end_row0 {
10781            let manual_hidden = state
10782                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
10783                .unwrap_or(false);
10784            let filter_hidden = state
10785                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
10786                .unwrap_or(false);
10787
10788            let include = match mode {
10789                VisibilityMaskMode::IncludeAll => true,
10790                VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
10791                VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
10792                VisibilityMaskMode::ExcludeManualOrFilterHidden => {
10793                    !(manual_hidden || filter_hidden)
10794                }
10795            };
10796            out.push(include);
10797        }
10798
10799        let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
10800        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
10801            const MAX_CACHE_ENTRIES: usize = 4096;
10802            if cache.len() >= MAX_CACHE_ENTRIES {
10803                cache.clear();
10804                #[cfg(test)]
10805                visibility_mask_test_hooks::inc_eviction();
10806            }
10807            cache.insert(key, mask.clone());
10808        }
10809
10810        Some(mask)
10811    }
10812
10813    fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
10814        let changes =
10815            crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
10816        let global_changed = changes.epoch != self.function_semantic_epoch_seen;
10817        let provider_revision = self.resolver.planning_semantic_revision();
10818        let provider_changed = provider_revision != self.function_provider_revision_seen;
10819        if !global_changed && !provider_changed {
10820            return Ok(false);
10821        }
10822
10823        let changed = !changes.keys.is_empty();
10824        if global_changed && changed || provider_changed {
10825            self.cached_static_schedule = None;
10826            self.recent_schedules.clear();
10827            self.base_schedule = None;
10828        }
10829        self.function_semantic_epoch_seen = changes.epoch;
10830        self.function_provider_revision_seen = provider_revision;
10831        Ok(false)
10832    }
10833
10834    pub(crate) fn ast_uses_changed_function(
10835        ast: &ASTNode,
10836        changed: &BTreeSet<(String, String)>,
10837    ) -> bool {
10838        match &ast.node_type {
10839            ASTNodeType::Function { name, args } => {
10840                let normalized = name.to_uppercase();
10841                let mut spellings = vec![(String::new(), normalized.clone())];
10842                let mut stripped = normalized.as_str();
10843                while let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
10844                    .iter()
10845                    .find_map(|prefix| stripped.strip_prefix(prefix))
10846                {
10847                    stripped = rest;
10848                    spellings.push((String::new(), stripped.to_string()));
10849                }
10850                let resolved = crate::function_registry::resolve("", name);
10851                let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
10852                    || resolved.as_ref().is_some_and(|resolved| {
10853                        changed.contains(&(
10854                            resolved.namespace.clone(),
10855                            resolved.canonical_name.clone(),
10856                        ))
10857                    });
10858                directly_changed
10859                    || resolved.is_none()
10860                    || args
10861                        .iter()
10862                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
10863            }
10864            ASTNodeType::Call { callee, args } => {
10865                Self::ast_uses_changed_function(callee, changed)
10866                    || args
10867                        .iter()
10868                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
10869            }
10870            ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
10871            ASTNodeType::BinaryOp { left, right, .. } => {
10872                Self::ast_uses_changed_function(left, changed)
10873                    || Self::ast_uses_changed_function(right, changed)
10874            }
10875            ASTNodeType::Array(rows) => rows
10876                .iter()
10877                .flatten()
10878                .any(|node| Self::ast_uses_changed_function(node, changed)),
10879            _ => false,
10880        }
10881    }
10882
10883    fn ast_contains_function(ast: &ASTNode) -> bool {
10884        match &ast.node_type {
10885            ASTNodeType::Function { .. } => true,
10886            ASTNodeType::Call { callee, args } => {
10887                Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
10888            }
10889            ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
10890            ASTNodeType::BinaryOp { left, right, .. } => {
10891                Self::ast_contains_function(left) || Self::ast_contains_function(right)
10892            }
10893            ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
10894            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
10895        }
10896    }
10897
10898    fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
10899        Region::rows_from(sheet_id, start_row0)
10900    }
10901
10902    fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
10903        Region::cols_from(sheet_id, start_col0)
10904    }
10905
10906    #[cfg(test)]
10907    pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
10908        self.force_non_cycle_schedule_fallback_for_test = true;
10909    }
10910
10911    fn materialize_deferred_sheet_before_structural_edit(
10912        &mut self,
10913        sheet: &str,
10914    ) -> Result<(), crate::engine::EditorError> {
10915        if self.staged_formulas.contains_key(sheet) {
10916            self.build_graph_for_sheets([sheet])?;
10917        }
10918        Ok(())
10919    }
10920
10921    fn structural_row_occupancy(
10922        &self,
10923        sheet: &str,
10924        sheet_id: SheetId,
10925    ) -> crate::engine::graph::StructuralOccupancy {
10926        if !self.graph.has_compressed_range_readers() {
10927            return crate::engine::graph::StructuralOccupancy::default();
10928        }
10929        let mut occupancy = self.graph.structural_occupancy(sheet_id);
10930        if let Some(arrow_sheet) = self.arrow_sheets.sheet(sheet) {
10931            occupancy.include_arrow_sheet(arrow_sheet);
10932            occupancy
10933        } else {
10934            // Missing Arrow state cannot prove an apparently empty column empty.
10935            crate::engine::graph::StructuralOccupancy::conservative()
10936        }
10937    }
10938
10939    fn structural_column_occupancy(&self) -> crate::engine::graph::StructuralOccupancy {
10940        // Arrow exposes occupied columns through chunk metadata and overlay maps,
10941        // but has no cheap occupied-row index. Column edits therefore deliberately
10942        // retain conservative cross-axis invalidation instead of scanning cells.
10943        crate::engine::graph::StructuralOccupancy::conservative()
10944    }
10945
10946    /// Insert rows (1-based) and mirror into Arrow store when enabled
10947    pub fn insert_rows(
10948        &mut self,
10949        sheet: &str,
10950        before: u32,
10951        count: u32,
10952    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
10953    {
10954        if count == 0 {
10955            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
10956        }
10957        self.observe_function_semantic_epoch()
10958            .map_err(crate::engine::EditorError::Excel)?;
10959        use crate::engine::graph::editor::vertex_editor::VertexEditor;
10960        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
10961        let sheet_id = self.ensure_known_sheet_id(sheet)?;
10962        let before0 = before.saturating_sub(1);
10963        let affected_region = Self::structural_row_region(sheet_id, before0);
10964        let occupancy = self.structural_row_occupancy(sheet, sheet_id);
10965        let summary = {
10966            let mut editor =
10967                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
10968            editor.insert_rows(sheet_id, before0, count)?
10969        };
10970        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
10971            let before0 = before0 as usize;
10972            asheet.insert_rows(before0, count as usize);
10973        }
10974        self.purge_derived_formats_after_row(sheet_id, before0);
10975        self.mark_moved_formula_vertices_dirty(&summary);
10976        self.clear_computed_overlay_after_row(sheet, before0 as usize);
10977        self.shift_row_visibility_insert(sheet_id, before0, count);
10978        self.record_structural_change(StructuralScope::Region(affected_region));
10979        self.mark_topology_edited();
10980        Ok(summary)
10981    }
10982
10983    /// Delete rows (1-based) and mirror into Arrow store when enabled
10984    pub fn delete_rows(
10985        &mut self,
10986        sheet: &str,
10987        start: u32,
10988        count: u32,
10989    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
10990    {
10991        if count == 0 {
10992            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
10993        }
10994        self.observe_function_semantic_epoch()
10995            .map_err(crate::engine::EditorError::Excel)?;
10996        use crate::engine::graph::editor::vertex_editor::VertexEditor;
10997        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
10998        let sheet_id = self.ensure_known_sheet_id(sheet)?;
10999        let start0 = start.saturating_sub(1);
11000        let affected_region = Self::structural_row_region(sheet_id, start0);
11001        let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11002        let summary = {
11003            let mut editor =
11004                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11005            editor.delete_rows(sheet_id, start0, count)?
11006        };
11007        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11008            let start0 = start0 as usize;
11009            asheet.delete_rows(start0, count as usize);
11010        }
11011        self.purge_derived_formats_after_row(sheet_id, start0);
11012        self.mark_moved_formula_vertices_dirty(&summary);
11013        self.clear_computed_overlay_after_row(sheet, start0 as usize);
11014        self.shift_row_visibility_delete(sheet_id, start0, count);
11015        self.record_structural_change(StructuralScope::Region(affected_region));
11016        self.mark_topology_edited();
11017        Ok(summary)
11018    }
11019
11020    /// Insert columns (1-based) and mirror into Arrow store when enabled
11021    pub fn insert_columns(
11022        &mut self,
11023        sheet: &str,
11024        before: u32,
11025        count: u32,
11026    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11027    {
11028        if count == 0 {
11029            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11030        }
11031        self.observe_function_semantic_epoch()
11032            .map_err(crate::engine::EditorError::Excel)?;
11033        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11034        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11035        let sheet_id = self.graph.sheet_id(sheet).ok_or(
11036            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11037                name: sheet.to_string(),
11038                reason: "Unknown sheet".to_string(),
11039            },
11040        )?;
11041        let before0 = before.saturating_sub(1);
11042        let affected_region = Self::structural_col_region(sheet_id, before0);
11043        let occupancy = self.structural_column_occupancy();
11044        let summary = {
11045            let mut editor =
11046                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11047            editor.insert_columns(sheet_id, before0, count)?
11048        };
11049        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11050            let before0 = before0 as usize;
11051            asheet.insert_columns(before0, count as usize);
11052        }
11053        self.purge_derived_formats_after_col(sheet_id, before0);
11054        self.mark_moved_formula_vertices_dirty(&summary);
11055        self.clear_computed_overlay_after_col(sheet, before0 as usize);
11056        self.record_structural_change(StructuralScope::Region(affected_region));
11057        self.mark_topology_edited();
11058        Ok(summary)
11059    }
11060
11061    /// Delete columns (1-based) and mirror into Arrow store when enabled
11062    pub fn delete_columns(
11063        &mut self,
11064        sheet: &str,
11065        start: u32,
11066        count: u32,
11067    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11068    {
11069        if count == 0 {
11070            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11071        }
11072        self.observe_function_semantic_epoch()
11073            .map_err(crate::engine::EditorError::Excel)?;
11074        use crate::engine::graph::editor::vertex_editor::VertexEditor;
11075        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11076        let sheet_id = self.graph.sheet_id(sheet).ok_or(
11077            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11078                name: sheet.to_string(),
11079                reason: "Unknown sheet".to_string(),
11080            },
11081        )?;
11082        let start0 = start.saturating_sub(1);
11083        let affected_region = Self::structural_col_region(sheet_id, start0);
11084        let occupancy = self.structural_column_occupancy();
11085        let summary = {
11086            let mut editor =
11087                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11088            editor.delete_columns(sheet_id, start0, count)?
11089        };
11090        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11091            let start0 = start0 as usize;
11092            asheet.delete_columns(start0, count as usize);
11093        }
11094        self.purge_derived_formats_after_col(sheet_id, start0);
11095        self.mark_moved_formula_vertices_dirty(&summary);
11096        self.clear_computed_overlay_after_col(sheet, start0 as usize);
11097        self.record_structural_change(StructuralScope::Region(affected_region));
11098        self.mark_topology_edited();
11099        Ok(summary)
11100    }
11101    /// Arrow-backed used row bounds across a column span (1-based inclusive cols).
11102    fn arrow_used_row_bounds(
11103        &self,
11104        sheet: &str,
11105        start_col: u32,
11106        end_col: u32,
11107    ) -> Option<(u32, u32)> {
11108        let a = self.sheet_store().sheet(sheet)?;
11109        if a.columns.is_empty() {
11110            return None;
11111        }
11112        let sc0 = start_col.saturating_sub(1) as usize;
11113        let ec0 = end_col.saturating_sub(1) as usize;
11114        let col_hi = a.columns.len().saturating_sub(1);
11115        if sc0 > col_hi {
11116            return None;
11117        }
11118        let ec0 = ec0.min(col_hi);
11119        // Pass-scoped cache with snapshot guard
11120        let snap = self.data_snapshot_id();
11121        let mut min_r0: Option<usize> = None;
11122        for ci in sc0..=ec0 {
11123            let sheet_id = self.graph.sheet_id(sheet)?;
11124            if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
11125                g.as_ref()
11126                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11127            }) {
11128                let mv = mv as usize;
11129                min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
11130                continue;
11131            }
11132            // Compute and store
11133            let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11134            if let Ok(mut g) = self.row_bounds_cache.write() {
11135                g.get_or_insert_with(|| RowBoundsCache::new(snap))
11136                    .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
11137            }
11138            if let Some(m) = min_c {
11139                min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
11140            }
11141        }
11142        min_r0?;
11143        let mut max_r0: Option<usize> = None;
11144        for ci in sc0..=ec0 {
11145            let sheet_id = self.graph.sheet_id(sheet)?;
11146            if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
11147                g.as_ref()
11148                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11149            }) {
11150                let mv = mv as usize;
11151                max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
11152                continue;
11153            }
11154            let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11155            if let Ok(mut g) = self.row_bounds_cache.write() {
11156                g.get_or_insert_with(|| RowBoundsCache::new(snap))
11157                    .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
11158            }
11159            if let Some(m) = max_c {
11160                max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
11161            }
11162        }
11163        match (min_r0, max_r0) {
11164            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11165            _ => None,
11166        }
11167    }
11168
11169    fn scan_column_used_bounds(
11170        a: &crate::arrow_store::ArrowSheet,
11171        ci: usize,
11172    ) -> (Option<u32>, Option<u32>) {
11173        let col = &a.columns[ci];
11174
11175        // Min: scan dense chunks first, then sparse chunks in ascending index order.
11176        let mut min_r0: Option<u32> = None;
11177        for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11178            let tags = chunk.type_tag.values();
11179            for (off, &t) in tags.iter().enumerate() {
11180                let overlay_non_empty = chunk
11181                    .overlay
11182                    .get(off)
11183                    .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11184                    .unwrap_or(false)
11185                    || chunk
11186                        .computed_overlay
11187                        .get(off)
11188                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11189                        .unwrap_or(false);
11190                if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11191                    let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11192                        break;
11193                    };
11194                    let row0 = chunk_start + off;
11195                    min_r0 = Some(row0 as u32);
11196                    break;
11197                }
11198            }
11199            if min_r0.is_some() {
11200                break;
11201            }
11202        }
11203        if min_r0.is_none() && !col.sparse_chunks.is_empty() {
11204            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11205            sparse_idxs.sort_unstable();
11206            for chunk_idx in sparse_idxs {
11207                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11208                    continue;
11209                };
11210                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11211                    continue;
11212                };
11213                let tags = chunk.type_tag.values();
11214                for (off, &t) in tags.iter().enumerate() {
11215                    let overlay_non_empty = chunk
11216                        .overlay
11217                        .get(off)
11218                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11219                        .unwrap_or(false)
11220                        || chunk
11221                            .computed_overlay
11222                            .get(off)
11223                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11224                            .unwrap_or(false);
11225                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11226                        let row0 = chunk_start + off;
11227                        min_r0 = Some(row0 as u32);
11228                        break;
11229                    }
11230                }
11231                if min_r0.is_some() {
11232                    break;
11233                }
11234            }
11235        }
11236
11237        // Max: scan sparse chunks in descending index order, then dense chunks in reverse.
11238        let mut max_r0: Option<u32> = None;
11239        if !col.sparse_chunks.is_empty() {
11240            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11241            sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
11242            for chunk_idx in sparse_idxs {
11243                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11244                    continue;
11245                };
11246                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11247                    continue;
11248                };
11249                let tags = chunk.type_tag.values();
11250                for (rev_idx, &t) in tags.iter().enumerate().rev() {
11251                    let overlay_non_empty = chunk
11252                        .overlay
11253                        .get(rev_idx)
11254                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11255                        .unwrap_or(false)
11256                        || chunk
11257                            .computed_overlay
11258                            .get(rev_idx)
11259                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11260                            .unwrap_or(false);
11261                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11262                        let row0 = chunk_start + rev_idx;
11263                        max_r0 = Some(row0 as u32);
11264                        break;
11265                    }
11266                }
11267                if max_r0.is_some() {
11268                    break;
11269                }
11270            }
11271        }
11272        if max_r0.is_none() {
11273            for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
11274                let tags = chunk.type_tag.values();
11275                for (rev_idx, &t) in tags.iter().enumerate().rev() {
11276                    let overlay_non_empty = chunk
11277                        .overlay
11278                        .get(rev_idx)
11279                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11280                        .unwrap_or(false)
11281                        || chunk
11282                            .computed_overlay
11283                            .get(rev_idx)
11284                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11285                            .unwrap_or(false);
11286                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11287                        let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11288                            break;
11289                        };
11290                        let row0 = chunk_start + rev_idx;
11291                        max_r0 = Some(row0 as u32);
11292                        break;
11293                    }
11294                }
11295                if max_r0.is_some() {
11296                    break;
11297                }
11298            }
11299        }
11300
11301        (min_r0, max_r0)
11302    }
11303
11304    /// Arrow-backed used column bounds across a row span (1-based inclusive rows).
11305    fn arrow_used_col_bounds(
11306        &self,
11307        sheet: &str,
11308        start_row: u32,
11309        end_row: u32,
11310    ) -> Option<(u32, u32)> {
11311        let a = self.sheet_store().sheet(sheet)?;
11312        if a.columns.is_empty() {
11313            return None;
11314        }
11315        let sr0 = start_row.saturating_sub(1) as usize;
11316        let er0 = end_row.saturating_sub(1) as usize;
11317        if sr0 > er0 {
11318            return None;
11319        }
11320        // Map start/end rows into chunk ranges
11321        // We will scan each column for any non-empty within [sr0..=er0]
11322        let mut min_c0: Option<usize> = None;
11323        let mut max_c0: Option<usize> = None;
11324        // Precompute chunk bounds for row range
11325        for (ci, col) in a.columns.iter().enumerate() {
11326            let mut any_in_range = false;
11327
11328            let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
11329                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11330                    return false;
11331                };
11332                let chunk_len = chunk.type_tag.len();
11333                if chunk_len == 0 {
11334                    return false;
11335                }
11336                let chunk_end = chunk_start + chunk_len.saturating_sub(1);
11337                // check intersection
11338                if sr0 > chunk_end || er0 < chunk_start {
11339                    return false;
11340                }
11341                let start_off = sr0.max(chunk_start) - chunk_start;
11342                let end_off = er0.min(chunk_end) - chunk_start;
11343                let tags = chunk.type_tag.values();
11344                for off in start_off..=end_off {
11345                    let overlay_non_empty = chunk
11346                        .overlay
11347                        .get(off)
11348                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11349                        .unwrap_or(false)
11350                        || chunk
11351                            .computed_overlay
11352                            .get(off)
11353                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11354                            .unwrap_or(false);
11355                    if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
11356                        return true;
11357                    }
11358                }
11359                false
11360            };
11361
11362            for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11363                if scan_chunk(chunk_idx, chunk) {
11364                    any_in_range = true;
11365                    break;
11366                }
11367            }
11368
11369            if !any_in_range && !col.sparse_chunks.is_empty() {
11370                for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
11371                    if scan_chunk(chunk_idx, chunk) {
11372                        any_in_range = true;
11373                        break;
11374                    }
11375                }
11376            }
11377
11378            if any_in_range {
11379                min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
11380                max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
11381            }
11382        }
11383        match (min_c0, max_c0) {
11384            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11385            _ => None,
11386        }
11387    }
11388
11389    fn formula_row_bounds_for_columns(
11390        &self,
11391        sheet: &str,
11392        start_col: u32,
11393        end_col: u32,
11394    ) -> Option<(u32, u32)> {
11395        let sheet_id = self.graph.sheet_id(sheet)?;
11396        let sc0 = start_col.saturating_sub(1);
11397        let ec0 = end_col.saturating_sub(1);
11398        let mut min_r0: Option<u32> = None;
11399        let mut max_r0: Option<u32> = None;
11400
11401        if self.graph.sheet_index(sheet_id).is_some() {
11402            for vid in self.graph.vertices_in_cols(sheet_id, sc0, ec0) {
11403                if !matches!(
11404                    self.graph.get_vertex_kind(vid),
11405                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11406                ) {
11407                    continue;
11408                }
11409                let Some(row0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.row()) else {
11410                    continue;
11411                };
11412                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11413                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11414            }
11415        } else {
11416            for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11417                if !matches!(
11418                    self.graph.get_vertex_kind(vid),
11419                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11420                ) {
11421                    continue;
11422                }
11423                let col0 = coord.col();
11424                if col0 < sc0 || col0 > ec0 {
11425                    continue;
11426                }
11427                let row0 = coord.row();
11428                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11429                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11430            }
11431        }
11432
11433        match (min_r0, max_r0) {
11434            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11435            _ => None,
11436        }
11437    }
11438
11439    fn formula_col_bounds_for_rows(
11440        &self,
11441        sheet: &str,
11442        start_row: u32,
11443        end_row: u32,
11444    ) -> Option<(u32, u32)> {
11445        let sheet_id = self.graph.sheet_id(sheet)?;
11446        let sr0 = start_row.saturating_sub(1);
11447        let er0 = end_row.saturating_sub(1);
11448        let mut min_c0: Option<u32> = None;
11449        let mut max_c0: Option<u32> = None;
11450
11451        if self.graph.sheet_index(sheet_id).is_some() {
11452            for vid in self.graph.vertices_in_rows(sheet_id, sr0, er0) {
11453                if !matches!(
11454                    self.graph.get_vertex_kind(vid),
11455                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11456                ) {
11457                    continue;
11458                }
11459                let Some(col0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.col()) else {
11460                    continue;
11461                };
11462                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11463                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11464            }
11465        } else {
11466            for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11467                if !matches!(
11468                    self.graph.get_vertex_kind(vid),
11469                    VertexKind::FormulaScalar | VertexKind::FormulaArray
11470                ) {
11471                    continue;
11472                }
11473                let row0 = coord.row();
11474                if row0 < sr0 || row0 > er0 {
11475                    continue;
11476                }
11477                let col0 = coord.col();
11478                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11479                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11480            }
11481        }
11482
11483        match (min_c0, max_c0) {
11484            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11485            _ => None,
11486        }
11487    }
11488
11489    fn union_used_bounds(
11490        first: Option<(u32, u32)>,
11491        second: Option<(u32, u32)>,
11492    ) -> Option<(u32, u32)> {
11493        match (first, second) {
11494            (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
11495            (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
11496            (None, None) => None,
11497        }
11498    }
11499
11500    /// Mirror a single cell value into the Arrow overlay if enabled.
11501    /// Handles capacity growth, per-chunk overlay set, and heuristic compaction.
11502    fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
11503        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11504            return;
11505        }
11506        if self.arrow_sheets.sheet(sheet).is_none() {
11507            self.arrow_sheets
11508                .sheets
11509                .push(crate::arrow_store::ArrowSheet {
11510                    name: std::sync::Arc::<str>::from(sheet),
11511                    date_system: self.config.date_system,
11512                    columns: Vec::new(),
11513                    nrows: 0,
11514                    chunk_starts: Vec::new(),
11515                    chunk_rows: 32 * 1024,
11516                });
11517        }
11518
11519        let row0 = row.saturating_sub(1) as usize;
11520        let col0 = col.saturating_sub(1) as usize;
11521
11522        let asheet = self
11523            .arrow_sheets
11524            .sheet_mut(sheet)
11525            .expect("ArrowSheet must exist");
11526
11527        let cur_cols = asheet.columns.len();
11528        if col0 >= cur_cols {
11529            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11530        }
11531
11532        if row0 >= asheet.nrows as usize {
11533            if asheet.columns.is_empty() {
11534                asheet.insert_columns(0, 1);
11535            }
11536            asheet.ensure_row_capacity(row0 + 1);
11537        }
11538        if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
11539            let ov =
11540                crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
11541            let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
11542                let _ = ch.overlay.set(in_off, ov);
11543                let format = match value {
11544                    LiteralValue::Date(_) => Some(crate::format::FormatId::DATE),
11545                    LiteralValue::DateTime(_) => Some(crate::format::FormatId::DATETIME),
11546                    LiteralValue::Time(_) => Some(crate::format::FormatId::TIME),
11547                    LiteralValue::Duration(_) => Some(crate::format::FormatId::DURATION),
11548                    _ => None,
11549                };
11550                ch.overlay.set_format(in_off, format);
11551                // A user edit must invalidate any computed (formula/spill) overlay entry at
11552                // this cell. Otherwise, if the delta overlay later compacts into the base lanes
11553                // (clearing `overlay`), a stale `computed_overlay=Empty` could incorrectly mask
11554                // the edited base value under the read cascade.
11555                ch.computed_overlay.remove(in_off)
11556            } else {
11557                return;
11558            };
11559            // Heuristic compaction: > len/50 or > 1024
11560            let abs_threshold = 1024usize;
11561            let frac_den = 50usize;
11562            let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
11563            if freed > 0 {
11564                self.overlay_compactions = self.overlay_compactions.saturating_add(1);
11565            }
11566            self.adjust_computed_overlay_bytes(computed_delta);
11567        }
11568    }
11569
11570    /// Remove a delta-overlay entry for a single cell (if present).
11571    ///
11572    /// This is used when transitioning a cell to a formula so that any previous user-edit overlay
11573    /// does not continue to mask computed overlay outputs.
11574    fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
11575        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11576            return;
11577        }
11578        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11579            return;
11580        };
11581        let row0 = row.saturating_sub(1) as usize;
11582        let col0 = col.saturating_sub(1) as usize;
11583        if row0 >= asheet.nrows as usize {
11584            return;
11585        }
11586        if col0 >= asheet.columns.len() {
11587            return;
11588        }
11589        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11590            return;
11591        };
11592        if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
11593            let _ = ch.overlay.remove(in_off);
11594        }
11595    }
11596
11597    fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
11598        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11599            return;
11600        }
11601
11602        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11603            return;
11604        };
11605        if start_row0 >= asheet.nrows as usize {
11606            return;
11607        }
11608
11609        let starts = asheet.chunk_starts.clone();
11610        let nrows = asheet.nrows as usize;
11611        let mut delta = 0isize;
11612        for col in &mut asheet.columns {
11613            for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
11614                let Some(&chunk_start) = starts.get(chunk_idx) else {
11615                    continue;
11616                };
11617                let chunk_end = starts
11618                    .get(chunk_idx + 1)
11619                    .copied()
11620                    .unwrap_or(nrows)
11621                    .min(chunk_start.saturating_add(ch.len()));
11622                if chunk_end <= start_row0 {
11623                    continue;
11624                }
11625                if chunk_start >= start_row0 {
11626                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11627                } else {
11628                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11629                    delta = delta
11630                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11631                }
11632            }
11633
11634            for (chunk_idx, ch) in &mut col.sparse_chunks {
11635                let Some(&chunk_start) = starts.get(*chunk_idx) else {
11636                    continue;
11637                };
11638                let chunk_end = starts
11639                    .get(*chunk_idx + 1)
11640                    .copied()
11641                    .unwrap_or(nrows)
11642                    .min(chunk_start.saturating_add(ch.len()));
11643                if chunk_end <= start_row0 {
11644                    continue;
11645                }
11646                if chunk_start >= start_row0 {
11647                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11648                } else {
11649                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11650                    delta = delta
11651                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11652                }
11653            }
11654        }
11655        self.adjust_computed_overlay_bytes(delta);
11656    }
11657
11658    fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
11659        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11660            return;
11661        }
11662
11663        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11664            return;
11665        };
11666        if start_col0 >= asheet.columns.len() {
11667            return;
11668        }
11669
11670        let mut delta = 0isize;
11671        for col in asheet.columns.iter_mut().skip(start_col0) {
11672            for ch in &mut col.chunks {
11673                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11674            }
11675            for ch in col.sparse_chunks.values_mut() {
11676                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11677            }
11678        }
11679        self.adjust_computed_overlay_bytes(delta);
11680    }
11681
11682    #[inline]
11683    fn literal_to_overlay_value(
11684        value: &LiteralValue,
11685        date_system: crate::engine::DateSystem,
11686    ) -> crate::arrow_store::OverlayValue {
11687        crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
11688    }
11689
11690    fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
11691        self.arrow_sheets
11692            .sheet(sheet)
11693            .map(|sheet| sheet.date_system)
11694            .unwrap_or(self.config.date_system)
11695    }
11696
11697    /// Read a single cell's delta overlay entry (if present), preserving the distinction between
11698    /// absent and explicit `Empty`.
11699    fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11700        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11701            return None;
11702        }
11703        let asheet = self.arrow_sheets.sheet(sheet)?;
11704        let row0 = row.saturating_sub(1) as usize;
11705        let col0 = col.saturating_sub(1) as usize;
11706        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11707            return None;
11708        }
11709        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
11710        let ch = asheet.columns[col0].chunk(ch_idx)?;
11711        ch.overlay
11712            .get_scalar(in_off)
11713            .map(|ov| ov.to_literal_for(asheet.date_system))
11714    }
11715
11716    /// Read a single cell's computed overlay entry (if present), preserving the distinction
11717    /// between absent and explicit `Empty`.
11718    fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11719        if !(self.config.arrow_storage_enabled
11720            && self.config.delta_overlay_enabled
11721            && self.config.write_formula_overlay_enabled)
11722        {
11723            return None;
11724        }
11725        let asheet = self.arrow_sheets.sheet(sheet)?;
11726        let row0 = row.saturating_sub(1) as usize;
11727        let col0 = col.saturating_sub(1) as usize;
11728        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11729            return None;
11730        }
11731        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
11732        let ch = asheet.columns[col0].chunk(ch_idx)?;
11733        ch.computed_overlay
11734            .get_scalar(in_off)
11735            .map(|ov| ov.to_literal_for(asheet.date_system))
11736    }
11737
11738    fn set_delta_overlay_cell_raw(
11739        &mut self,
11740        sheet: &str,
11741        row: u32,
11742        col: u32,
11743        value: Option<LiteralValue>,
11744    ) {
11745        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11746            return;
11747        }
11748
11749        self.ensure_arrow_sheet(sheet);
11750        let date_system = self.arrow_sheet_date_system(sheet);
11751        let ov_opt = value
11752            .as_ref()
11753            .map(|value| Self::literal_to_overlay_value(value, date_system));
11754        let row0 = row.saturating_sub(1) as usize;
11755        let col0 = col.saturating_sub(1) as usize;
11756        let asheet = self
11757            .arrow_sheets
11758            .sheet_mut(sheet)
11759            .expect("ArrowSheet must exist");
11760
11761        let cur_cols = asheet.columns.len();
11762        if col0 >= cur_cols {
11763            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11764        }
11765        if row0 >= asheet.nrows as usize {
11766            if asheet.columns.is_empty() {
11767                asheet.insert_columns(0, 1);
11768            }
11769            asheet.ensure_row_capacity(row0 + 1);
11770        }
11771
11772        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11773            return;
11774        };
11775        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
11776            return;
11777        };
11778
11779        if let Some(ov) = ov_opt {
11780            let _ = ch.overlay.set(in_off, ov);
11781        } else {
11782            let _ = ch.overlay.remove(in_off);
11783        }
11784    }
11785
11786    fn set_computed_overlay_cell_raw(
11787        &mut self,
11788        sheet: &str,
11789        row: u32,
11790        col: u32,
11791        value: Option<LiteralValue>,
11792    ) {
11793        if !(self.config.arrow_storage_enabled
11794            && self.config.delta_overlay_enabled
11795            && self.config.write_formula_overlay_enabled)
11796        {
11797            return;
11798        }
11799
11800        self.ensure_arrow_sheet(sheet);
11801        let date_system = self.arrow_sheet_date_system(sheet);
11802        let ov_opt = value
11803            .as_ref()
11804            .map(|value| Self::literal_to_overlay_value(value, date_system));
11805        let row0 = row.saturating_sub(1) as usize;
11806        let col0 = col.saturating_sub(1) as usize;
11807        let asheet = self
11808            .arrow_sheets
11809            .sheet_mut(sheet)
11810            .expect("ArrowSheet must exist");
11811
11812        let cur_cols = asheet.columns.len();
11813        if col0 >= cur_cols {
11814            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11815        }
11816        if row0 >= asheet.nrows as usize {
11817            if asheet.columns.is_empty() {
11818                asheet.insert_columns(0, 1);
11819            }
11820            asheet.ensure_row_capacity(row0 + 1);
11821        }
11822
11823        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11824            return;
11825        };
11826        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
11827            return;
11828        };
11829
11830        let delta = if let Some(ov) = ov_opt {
11831            ch.computed_overlay.set(in_off, ov)
11832        } else {
11833            ch.computed_overlay.remove(in_off)
11834        };
11835        self.adjust_computed_overlay_bytes(delta);
11836    }
11837
11838    fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
11839        use crate::engine::ArrowOp;
11840
11841        let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
11842            Box::new(batch.ops.iter().rev())
11843        } else {
11844            Box::new(batch.ops.iter())
11845        };
11846
11847        for op in iter {
11848            match op {
11849                ArrowOp::SetDeltaCell {
11850                    sheet_id,
11851                    row0,
11852                    col0,
11853                    old,
11854                    new,
11855                } => {
11856                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
11857                    let v = if undo { old.clone() } else { new.clone() };
11858                    self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
11859                }
11860                ArrowOp::SetComputedCell {
11861                    sheet_id,
11862                    row0,
11863                    col0,
11864                    old,
11865                    new,
11866                } => {
11867                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
11868                    let v = if undo { old.clone() } else { new.clone() };
11869                    self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
11870                }
11871                ArrowOp::RestoreComputedRect {
11872                    sheet_id,
11873                    sr0,
11874                    sc0,
11875                    er0,
11876                    ec0,
11877                    old,
11878                    new,
11879                } => {
11880                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
11881                    let vals = if undo { old } else { new };
11882                    let height = (*er0).saturating_sub(*sr0) as usize + 1;
11883                    let width = (*ec0).saturating_sub(*sc0) as usize + 1;
11884                    for r in 0..height {
11885                        for c in 0..width {
11886                            let v = vals
11887                                .get(r)
11888                                .and_then(|row| row.get(c))
11889                                .cloned()
11890                                .unwrap_or(LiteralValue::Empty);
11891                            self.set_computed_overlay_cell_raw(
11892                                &sheet,
11893                                *sr0 + 1 + r as u32,
11894                                *sc0 + 1 + c as u32,
11895                                Some(v),
11896                            );
11897                        }
11898                    }
11899                }
11900                ArrowOp::InsertRows {
11901                    sheet_id,
11902                    before0,
11903                    count,
11904                } => {
11905                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
11906                    self.ensure_arrow_sheet(&sheet);
11907                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
11908                        if undo {
11909                            asheet.delete_rows(*before0 as usize, *count as usize);
11910                        } else {
11911                            asheet.insert_rows(*before0 as usize, *count as usize);
11912                        }
11913                    }
11914                    self.purge_derived_formats_after_row(*sheet_id, *before0);
11915                }
11916                ArrowOp::InsertCols {
11917                    sheet_id,
11918                    before0,
11919                    count,
11920                } => {
11921                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
11922                    self.ensure_arrow_sheet(&sheet);
11923                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
11924                        if undo {
11925                            asheet.delete_columns(*before0 as usize, *count as usize);
11926                        } else {
11927                            asheet.insert_columns(*before0 as usize, *count as usize);
11928                        }
11929                    }
11930                    self.purge_derived_formats_after_col(*sheet_id, *before0);
11931                }
11932            }
11933        }
11934    }
11935
11936    fn record_spill_ops_into_arrow_undo(
11937        &mut self,
11938        undo: &mut crate::engine::ArrowUndoBatch,
11939        events: &[crate::engine::ChangeEvent],
11940    ) {
11941        use crate::engine::ChangeEvent;
11942        use formualizer_common::LiteralValue;
11943
11944        #[allow(clippy::type_complexity)]
11945        let rect_from_snapshot =
11946            |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
11947             -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
11948                if snap.target_cells.is_empty() {
11949                    return None;
11950                }
11951                let sheet_id = snap.target_cells[0].sheet_id;
11952                let sr0 = snap.target_cells[0].coord.row();
11953                let sc0 = snap.target_cells[0].coord.col();
11954                if snap.values.is_empty() || snap.values[0].is_empty() {
11955                    return None;
11956                }
11957                let h = snap.values.len() as u32;
11958                let w = snap.values[0].len() as u32;
11959                let er0 = sr0.saturating_add(h.saturating_sub(1));
11960                let ec0 = sc0.saturating_add(w.saturating_sub(1));
11961                Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
11962            };
11963
11964        for ev in events {
11965            match ev {
11966                ChangeEvent::SpillCommitted { old, new, .. } => {
11967                    if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
11968                        let old_vals = if let Some(old_snap) = old {
11969                            rect_from_snapshot(old_snap)
11970                                .map(|(_, _, _, _, _, v)| v)
11971                                .unwrap_or_else(|| {
11972                                    vec![
11973                                        vec![LiteralValue::Empty; new_vals[0].len()];
11974                                        new_vals.len()
11975                                    ]
11976                                })
11977                        } else {
11978                            vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
11979                        };
11980                        undo.record_restore_computed_rect(
11981                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
11982                        );
11983                    }
11984                }
11985                ChangeEvent::SpillCleared { old, .. } => {
11986                    if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
11987                        let new_vals =
11988                            vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
11989                        undo.record_restore_computed_rect(
11990                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
11991                        );
11992                    }
11993                }
11994                _ => {}
11995            }
11996        }
11997    }
11998
11999    /// Mirror a value into the computed overlay (formula/spill outputs).
12000    ///
12001    /// This path is subject to `EvalConfig.max_overlay_memory_bytes`.
12002    /// If the cap is exceeded, computed overlays are compacted into base lanes.
12003    fn mirror_value_to_computed_overlay(
12004        &mut self,
12005        sheet: &str,
12006        row: u32,
12007        col: u32,
12008        value: &LiteralValue,
12009    ) {
12010        if !(self.config.arrow_storage_enabled
12011            && self.config.delta_overlay_enabled
12012            && self.config.write_formula_overlay_enabled)
12013        {
12014            return;
12015        }
12016        if self.computed_overlay_mirroring_disabled {
12017            return;
12018        }
12019
12020        let date_system = self.arrow_sheet_date_system(sheet);
12021        let ov = Self::literal_to_overlay_value(value, date_system);
12022        self.write_computed_overlay_value_0based(
12023            sheet,
12024            row.saturating_sub(1),
12025            col.saturating_sub(1),
12026            ov,
12027        );
12028    }
12029
12030    fn record_derived_format(&self, vertex_id: VertexId, format: Option<crate::format::FormatId>) {
12031        if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
12032            self.record_derived_format_at(cell, format);
12033        }
12034    }
12035
12036    fn record_derived_format_at(&self, cell: CellRef, format: Option<crate::format::FormatId>) {
12037        #[cfg(test)]
12038        self.derived_format_operations_for_test
12039            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12040        let format = format.filter(|id| *id != crate::format::FormatId::GENERAL);
12041        self.derived_formats.set(cell, format);
12042    }
12043
12044    fn clear_cell_format_state(&mut self, sheet: &str, cell: CellRef) {
12045        self.derived_formats.set(cell, None);
12046        if let Some(arrow) = self.arrow_sheets.sheet_mut(sheet) {
12047            arrow.clear_format(cell.coord.row() as usize, cell.coord.col() as usize);
12048        }
12049    }
12050
12051    fn clear_logged_cell_format_states(&mut self, events: &[ChangeEvent]) {
12052        let cells = events
12053            .iter()
12054            .filter_map(|event| match event {
12055                ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
12056                    Some(*addr)
12057                }
12058                _ => None,
12059            })
12060            .collect::<FxHashSet<_>>();
12061        for cell in cells {
12062            let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12063            self.clear_cell_format_state(&sheet, cell);
12064        }
12065    }
12066
12067    fn purge_derived_formats_after_row(&mut self, sheet_id: SheetId, start0: u32) {
12068        self.derived_formats
12069            .retain(|cell| cell.sheet_id != sheet_id || cell.coord.row() < start0);
12070    }
12071
12072    fn purge_derived_formats_after_col(&mut self, sheet_id: SheetId, start0: u32) {
12073        self.derived_formats
12074            .retain(|cell| cell.sheet_id != sheet_id || cell.coord.col() < start0);
12075    }
12076
12077    fn purge_derived_formats_for_sheet(&mut self, sheet_id: SheetId) {
12078        self.derived_formats
12079            .retain(|cell| cell.sheet_id != sheet_id);
12080    }
12081
12082    #[cfg(test)]
12083    pub(crate) fn debug_computed_overlay_format_0based(
12084        &self,
12085        sheet: &str,
12086        row0: u32,
12087        col0: u32,
12088    ) -> Option<crate::format::FormatId> {
12089        let sheet = self.arrow_sheets.sheet(sheet)?;
12090        let (chunk_idx, row_in_chunk) = sheet.chunk_of_row(row0 as usize)?;
12091        sheet
12092            .columns
12093            .get(col0 as usize)?
12094            .chunk(chunk_idx)?
12095            .computed_overlay
12096            .get_format(row_in_chunk)
12097    }
12098
12099    #[cfg(test)]
12100    pub(crate) fn debug_computed_overlay_chunk_has_formats_0based(
12101        &self,
12102        sheet: &str,
12103        row0: u32,
12104        col0: u32,
12105    ) -> bool {
12106        let Some(sheet) = self.arrow_sheets.sheet(sheet) else {
12107            return false;
12108        };
12109        let Some((chunk_idx, _)) = sheet.chunk_of_row(row0 as usize) else {
12110            return false;
12111        };
12112        sheet
12113            .columns
12114            .get(col0 as usize)
12115            .and_then(|column| column.chunk(chunk_idx))
12116            .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12117    }
12118
12119    #[cfg(test)]
12120    pub(crate) fn debug_clear_derived_format_0based(&mut self, sheet: &str, row0: u32, col0: u32) {
12121        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12122            self.derived_formats
12123                .set(CellRef::new_absolute(sheet_id, row0, col0), None);
12124        }
12125    }
12126
12127    #[cfg(test)]
12128    pub(crate) fn debug_record_derived_format_0based(
12129        &self,
12130        sheet: &str,
12131        row0: u32,
12132        col0: u32,
12133        format: Option<crate::format::FormatId>,
12134    ) {
12135        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12136            self.record_derived_format_at(CellRef::new_absolute(sheet_id, row0, col0), format);
12137        }
12138    }
12139
12140    #[cfg(test)]
12141    pub(crate) fn debug_derived_format_0based(
12142        &self,
12143        sheet: &str,
12144        row0: u32,
12145        col0: u32,
12146    ) -> Option<crate::format::FormatId> {
12147        let sheet_id = self.graph.sheet_id(sheet)?;
12148        self.derived_formats
12149            .get(&CellRef::new_absolute(sheet_id, row0, col0))
12150    }
12151
12152    #[cfg(test)]
12153    pub(crate) fn debug_reset_format_write_operation_counts(&mut self) {
12154        self.derived_format_operations_for_test
12155            .store(0, std::sync::atomic::Ordering::Relaxed);
12156        self.computed_overlay_set_explicit_entry_operations_for_test = 0;
12157        self.computed_overlay_stale_clear_range_effects_for_test = 0;
12158        self.computed_overlay_stale_clear_offset_attempts_for_test = 0;
12159        self.computed_format_vector_allocations_for_test
12160            .store(0, std::sync::atomic::Ordering::Relaxed);
12161    }
12162
12163    #[cfg(test)]
12164    pub(crate) fn debug_format_write_operation_counts(&self) -> (u64, u64, u64, u64, u64) {
12165        (
12166            self.derived_format_operations_for_test
12167                .load(std::sync::atomic::Ordering::Relaxed),
12168            self.computed_overlay_set_explicit_entry_operations_for_test,
12169            self.computed_format_vector_allocations_for_test
12170                .load(std::sync::atomic::Ordering::Relaxed),
12171            self.computed_overlay_stale_clear_range_effects_for_test,
12172            self.computed_overlay_stale_clear_offset_attempts_for_test,
12173        )
12174    }
12175
12176    fn write_computed_overlay_format_0based(
12177        &mut self,
12178        sheet: &str,
12179        row0: u32,
12180        col0: u32,
12181        format: Option<crate::format::FormatId>,
12182    ) {
12183        self.ensure_arrow_sheet(sheet);
12184        let (row0, col0) = (row0 as usize, col0 as usize);
12185        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
12186            return;
12187        };
12188        if col0 >= asheet.columns.len() {
12189            asheet.insert_columns(asheet.columns.len(), col0 + 1 - asheet.columns.len());
12190        }
12191        if row0 >= asheet.nrows as usize {
12192            asheet.ensure_row_capacity(row0 + 1);
12193        }
12194        let Some((chunk, offset)) = asheet.chunk_of_row(row0) else {
12195            return;
12196        };
12197        if let Some(chunk) = asheet.ensure_column_chunk_mut(col0, chunk) {
12198            chunk.computed_overlay.set_format(offset, format);
12199        }
12200    }
12201
12202    /// One unbuffered computed write of `value` and its derived `format` at
12203    /// `cell` (a single sheet lookup; the same writes as
12204    /// `write_computed_overlay_value_0based` then
12205    /// `write_computed_overlay_format_0based`).
12206    fn write_computed_cell_0based(
12207        &mut self,
12208        cell: CellRef,
12209        value: &LiteralValue,
12210        format: Option<crate::format::FormatId>,
12211    ) {
12212        if !(self.config.arrow_storage_enabled
12213            && self.config.delta_overlay_enabled
12214            && self.config.write_formula_overlay_enabled)
12215            || self.computed_overlay_mirroring_disabled
12216        {
12217            return;
12218        }
12219        let sheet = self.graph.sheet_name(cell.sheet_id);
12220        let index = match self
12221            .arrow_sheets
12222            .sheets
12223            .iter()
12224            .position(|s| s.name.as_ref() == sheet)
12225        {
12226            Some(index) => index,
12227            None => {
12228                let sheet = sheet.to_string();
12229                self.ensure_arrow_sheet(&sheet);
12230                self.arrow_sheets.sheets.len() - 1
12231            }
12232        };
12233        let (row0, col0) = (cell.coord.row() as usize, cell.coord.col() as usize);
12234        let asheet = &mut self.arrow_sheets.sheets[index];
12235        let ov = Self::literal_to_overlay_value(value, asheet.date_system);
12236        let cur_cols = asheet.columns.len();
12237        if col0 >= cur_cols {
12238            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12239        }
12240        if row0 >= asheet.nrows as usize {
12241            asheet.ensure_row_capacity(row0 + 1);
12242        }
12243        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12244            return;
12245        };
12246        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12247            return;
12248        };
12249        let delta = ch.computed_overlay.set_scalar(in_off, ov);
12250        ch.computed_overlay.set_format(in_off, format);
12251        self.adjust_computed_overlay_bytes(delta);
12252        if let Some(cap) = self.config.max_overlay_memory_bytes
12253            && self.computed_overlay_bytes_estimate > cap
12254        {
12255            self.disable_computed_overlay_mirroring_due_to_budget(cap);
12256        }
12257    }
12258
12259    fn write_computed_overlay_value_0based(
12260        &mut self,
12261        sheet: &str,
12262        row0: u32,
12263        col0: u32,
12264        value: OverlayValue,
12265    ) {
12266        if !(self.config.arrow_storage_enabled
12267            && self.config.delta_overlay_enabled
12268            && self.config.write_formula_overlay_enabled)
12269        {
12270            return;
12271        }
12272        if self.computed_overlay_mirroring_disabled {
12273            return;
12274        }
12275
12276        self.ensure_arrow_sheet(sheet);
12277
12278        let row0 = row0 as usize;
12279        let col0 = col0 as usize;
12280        let asheet = self
12281            .arrow_sheets
12282            .sheet_mut(sheet)
12283            .expect("ArrowSheet must exist");
12284
12285        let cur_cols = asheet.columns.len();
12286        if col0 >= cur_cols {
12287            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12288        }
12289
12290        if row0 >= asheet.nrows as usize {
12291            if asheet.columns.is_empty() {
12292                asheet.insert_columns(0, 1);
12293            }
12294            asheet.ensure_row_capacity(row0 + 1);
12295        }
12296
12297        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12298            return;
12299        };
12300        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12301            return;
12302        };
12303
12304        let delta = ch.computed_overlay.set_scalar(in_off, value);
12305        self.adjust_computed_overlay_bytes(delta);
12306
12307        if let Some(cap) = self.config.max_overlay_memory_bytes
12308            && self.computed_overlay_bytes_estimate > cap
12309        {
12310            self.disable_computed_overlay_mirroring_due_to_budget(cap);
12311        }
12312    }
12313
12314    pub(crate) fn plan_computed_write_coalescing(
12315        &self,
12316        buffer: &ComputedWriteBuffer,
12317    ) -> ComputedWriteCoalescingPlan {
12318        self.plan_computed_write_coalescing_from_writes(
12319            buffer.writes().iter().cloned(),
12320            buffer.formats_present,
12321        )
12322    }
12323
12324    fn plan_owned_computed_write_coalescing(
12325        &self,
12326        writes: Vec<ComputedWrite>,
12327        formats_present: bool,
12328    ) -> ComputedWriteCoalescingPlan {
12329        self.plan_computed_write_coalescing_from_writes(writes, formats_present)
12330    }
12331
12332    fn plan_computed_write_coalescing_from_writes(
12333        &self,
12334        writes: impl IntoIterator<Item = ComputedWrite>,
12335        formats_present: bool,
12336    ) -> ComputedWriteCoalescingPlan {
12337        let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
12338            BTreeMap::new();
12339        let mut input_cells = 0usize;
12340        // One sheet lookup per sheet run of writes, not per cell.
12341        let mut located: Option<(SheetId, Option<&crate::arrow_store::ArrowSheet>)> = None;
12342
12343        for write in writes {
12344            match write {
12345                ComputedWrite::Cell {
12346                    seq,
12347                    sheet_id,
12348                    row0,
12349                    col0,
12350                    value,
12351                    format_id,
12352                } => {
12353                    input_cells = input_cells.saturating_add(1);
12354                    let sheet = match located {
12355                        Some((id, sheet)) if id == sheet_id => sheet,
12356                        _ => {
12357                            let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12358                            located = Some((sheet_id, sheet));
12359                            sheet
12360                        }
12361                    };
12362                    let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12363                        Some(sheet) => {
12364                            Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize)
12365                        }
12366                        None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12367                            row0 as usize,
12368                            32 * 1024,
12369                        ),
12370                    };
12371                    groups
12372                        .entry(ComputedWriteChunkKey {
12373                            sheet_id,
12374                            col0,
12375                            chunk_idx,
12376                            chunk_start_row0,
12377                        })
12378                        .or_default()
12379                        .push(ComputedWriteChunkEntryPlan {
12380                            row_in_chunk,
12381                            seq,
12382                            value,
12383                            format_id,
12384                        });
12385                }
12386                ComputedWrite::Run {
12387                    seq,
12388                    sheet_id,
12389                    row0,
12390                    col0,
12391                    entries,
12392                } => {
12393                    input_cells = input_cells.saturating_add(entries.len());
12394                    let sheet = match located {
12395                        Some((id, sheet)) if id == sheet_id => sheet,
12396                        _ => {
12397                            let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12398                            located = Some((sheet_id, sheet));
12399                            sheet
12400                        }
12401                    };
12402                    // Rows are located one by one (a binary search, no sheet
12403                    // lookup); the group map is touched once per chunk
12404                    // segment of the run.
12405                    let mut segment: Vec<ComputedWriteChunkEntryPlan> = Vec::new();
12406                    let mut segment_key: Option<ComputedWriteChunkKey> = None;
12407                    for (k, (value, format_id)) in entries.into_iter().enumerate() {
12408                        let row = row0.saturating_add(k as u32) as usize;
12409                        let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12410                            Some(sheet) => {
12411                                Self::locate_row_in_sheet_for_computed_write_plan(sheet, row)
12412                            }
12413                            None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12414                                row,
12415                                32 * 1024,
12416                            ),
12417                        };
12418                        let key = ComputedWriteChunkKey {
12419                            sheet_id,
12420                            col0,
12421                            chunk_idx,
12422                            chunk_start_row0,
12423                        };
12424                        if segment_key != Some(key)
12425                            && let Some(done) = segment_key.replace(key)
12426                        {
12427                            groups.entry(done).or_default().append(&mut segment);
12428                        }
12429                        segment.push(ComputedWriteChunkEntryPlan {
12430                            row_in_chunk,
12431                            seq,
12432                            value,
12433                            format_id,
12434                        });
12435                    }
12436                    if let Some(done) = segment_key {
12437                        groups.entry(done).or_default().append(&mut segment);
12438                    }
12439                }
12440                ComputedWrite::Rect {
12441                    seq,
12442                    sheet_id,
12443                    sr0,
12444                    sc0,
12445                    values,
12446                } => {
12447                    for (r_off, row) in values.into_iter().enumerate() {
12448                        for (c_off, value) in row.into_iter().enumerate() {
12449                            input_cells = input_cells.saturating_add(1);
12450                            self.push_computed_write_plan_entry(
12451                                &mut groups,
12452                                seq,
12453                                sheet_id,
12454                                sr0.saturating_add(r_off as u32),
12455                                sc0.saturating_add(c_off as u32),
12456                                value,
12457                                None,
12458                            );
12459                        }
12460                    }
12461                }
12462            }
12463        }
12464
12465        let mut plan = ComputedWriteCoalescingPlan {
12466            chunks: Vec::with_capacity(groups.len()),
12467            input_cells,
12468            coalesced_cells: 0,
12469            overwritten_cells: 0,
12470        };
12471        for (key, entries) in groups {
12472            let computed_lane_has_formats =
12473                self.computed_overlay_chunk_has_formats(key.sheet_id, key.col0, key.chunk_idx);
12474            let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(
12475                key,
12476                entries,
12477                formats_present,
12478                computed_lane_has_formats,
12479            );
12480            #[cfg(test)]
12481            if matches!(
12482                &chunk_plan.format_effect,
12483                ComputedWriteChunkFormatEffect::SetExplicit(_)
12484            ) {
12485                self.computed_format_vector_allocations_for_test
12486                    .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12487            }
12488            plan.coalesced_cells = plan
12489                .coalesced_cells
12490                .saturating_add(chunk_plan.entries.len());
12491            plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
12492            plan.chunks.push(chunk_plan);
12493        }
12494        debug_assert_eq!(
12495            plan.input_cells,
12496            plan.coalesced_cells.saturating_add(plan.overwritten_cells)
12497        );
12498        plan
12499    }
12500
12501    fn push_computed_write_plan_entry(
12502        &self,
12503        groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
12504        seq: u64,
12505        sheet_id: SheetId,
12506        row0: u32,
12507        col0: u32,
12508        value: OverlayValue,
12509        format_id: Option<crate::format::FormatId>,
12510    ) {
12511        let (chunk_idx, chunk_start_row0, row_in_chunk) =
12512            self.locate_computed_write_chunk(sheet_id, row0);
12513        let key = ComputedWriteChunkKey {
12514            sheet_id,
12515            col0,
12516            chunk_idx,
12517            chunk_start_row0,
12518        };
12519        groups
12520            .entry(key)
12521            .or_default()
12522            .push(ComputedWriteChunkEntryPlan {
12523                row_in_chunk,
12524                seq,
12525                value,
12526                format_id,
12527            });
12528    }
12529
12530    fn computed_overlay_chunk_has_formats(
12531        &self,
12532        sheet_id: SheetId,
12533        col0: u32,
12534        chunk_idx: usize,
12535    ) -> bool {
12536        let sheet_name = self.graph.sheet_name(sheet_id);
12537        self.arrow_sheets
12538            .sheet(sheet_name)
12539            .and_then(|sheet| sheet.columns.get(col0 as usize))
12540            .and_then(|column| column.chunk(chunk_idx))
12541            .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12542    }
12543
12544    fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
12545        let sheet_name = self.graph.sheet_name(sheet_id);
12546        if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
12547            return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
12548        }
12549        Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
12550    }
12551
12552    fn locate_row_in_sheet_for_computed_write_plan(
12553        sheet: &crate::arrow_store::ArrowSheet,
12554        row0: usize,
12555    ) -> (usize, u32, usize) {
12556        if row0 < sheet.nrows as usize
12557            && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
12558        {
12559            let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
12560            return (chunk_idx, chunk_start as u32, row_in_chunk);
12561        }
12562
12563        let chunk_rows = sheet.chunk_rows.max(1);
12564        if sheet.chunk_starts.is_empty() {
12565            return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
12566        }
12567
12568        let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
12569        let mut chunk_start = sheet.chunk_starts[chunk_idx];
12570        while chunk_start.saturating_add(chunk_rows) <= row0 {
12571            chunk_idx = chunk_idx.saturating_add(1);
12572            chunk_start = chunk_start.saturating_add(chunk_rows);
12573        }
12574        (
12575            chunk_idx,
12576            chunk_start as u32,
12577            row0.saturating_sub(chunk_start),
12578        )
12579    }
12580
12581    fn locate_row_in_empty_sheet_for_computed_write_plan(
12582        row0: usize,
12583        chunk_rows: usize,
12584    ) -> (usize, u32, usize) {
12585        let chunk_rows = chunk_rows.max(1);
12586        let chunk_idx = row0 / chunk_rows;
12587        let chunk_start = chunk_idx.saturating_mul(chunk_rows);
12588        (
12589            chunk_idx,
12590            chunk_start as u32,
12591            row0.saturating_sub(chunk_start),
12592        )
12593    }
12594
12595    #[cfg(test)]
12596    pub(crate) fn debug_plan_computed_write_coalescing(
12597        &self,
12598        buffer: &ComputedWriteBuffer,
12599    ) -> ComputedWriteCoalescingPlan {
12600        self.plan_computed_write_coalescing(buffer)
12601    }
12602
12603    pub(crate) fn flush_computed_write_buffer(
12604        &mut self,
12605        buffer: &mut ComputedWriteBuffer,
12606    ) -> Result<(), ExcelError> {
12607        if buffer.is_empty() {
12608            return Ok(());
12609        }
12610
12611        // Keep ownership of all pending writes until the final request
12612        // checkpoint and bounded commit-window preflight succeed so failures
12613        // remain retry safe. The synchronization and flush that follow are
12614        // infallible mutations with no cancellation point.
12615        self.resource_checkpoint(0)?;
12616        let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
12617        let (writes, formats_present) = buffer.take_writes();
12618        let plan = self.plan_owned_computed_write_coalescing(writes, formats_present);
12619        self.flush_computed_write_plan(plan);
12620        self.observe_evaluation_commit_window(commit_started);
12621
12622        Ok(())
12623    }
12624
12625    fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
12626        for chunk in plan.chunks {
12627            self.flush_computed_write_chunk_plan(chunk);
12628        }
12629    }
12630
12631    fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
12632        match &chunk.shape {
12633            ComputedWriteChunkPlanShape::Point => {
12634                self.flush_computed_write_chunk_plan_as_points(chunk);
12635            }
12636            ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
12637                self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
12638            }
12639            ComputedWriteChunkPlanShape::DenseRange { .. } => {
12640                self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12641            }
12642            ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
12643                if Self::should_emit_computed_run_fragment(*len, *runs) {
12644                    self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
12645                } else {
12646                    self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12647                }
12648            }
12649        }
12650    }
12651
12652    #[inline]
12653    fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
12654        runs <= len / 2
12655    }
12656
12657    fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
12658        let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
12659        for entry in chunk.entries {
12660            let row0 = chunk
12661                .chunk_start_row0
12662                .saturating_add(entry.row_in_chunk as u32);
12663            self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
12664        }
12665        self.apply_computed_overlay_format_effect(
12666            chunk.sheet_id,
12667            chunk.col0,
12668            chunk.chunk_idx,
12669            chunk.format_effect,
12670        );
12671    }
12672
12673    fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
12674        &mut self,
12675        chunk: ComputedWriteChunkPlan,
12676    ) {
12677        let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
12678        let sheet_id = chunk.sheet_id;
12679        let col0 = chunk.col0;
12680        let chunk_idx = chunk.chunk_idx;
12681        let chunk_start_row0 = chunk.chunk_start_row0;
12682        let format_effect = chunk.format_effect;
12683        let items: Vec<(usize, OverlayValue)> = chunk
12684            .entries
12685            .into_iter()
12686            .map(|entry| (entry.row_in_chunk, entry.value))
12687            .collect();
12688        match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
12689            items,
12690            point_estimate,
12691        ) {
12692            Some(Ok(fragment)) => {
12693                self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
12694            }
12695            Some(Err(cells)) => {
12696                self.flush_computed_overlay_cells_as_points(
12697                    sheet_id,
12698                    col0,
12699                    chunk_start_row0,
12700                    cells,
12701                );
12702            }
12703            None => {}
12704        }
12705        self.apply_computed_overlay_format_effect(sheet_id, col0, chunk_idx, format_effect);
12706    }
12707
12708    #[inline]
12709    fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
12710        chunk
12711            .entries
12712            .iter()
12713            .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
12714            .fold(0usize, usize::saturating_add)
12715    }
12716
12717    fn flush_computed_overlay_cells_as_points(
12718        &mut self,
12719        sheet_id: SheetId,
12720        col0: u32,
12721        chunk_start_row0: u32,
12722        cells: Vec<(usize, OverlayValue)>,
12723    ) {
12724        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
12725        for (row_in_chunk, value) in cells {
12726            let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
12727            self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
12728        }
12729    }
12730
12731    fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
12732        if chunk.entries.is_empty() {
12733            return;
12734        }
12735        let start = chunk.entries[0].row_in_chunk;
12736        let values: Vec<OverlayValue> =
12737            chunk.entries.into_iter().map(|entry| entry.value).collect();
12738        if let Some(fragment) = OverlayFragment::dense_range(start, values) {
12739            self.apply_computed_overlay_fragment(
12740                chunk.sheet_id,
12741                chunk.col0,
12742                chunk.chunk_idx,
12743                fragment,
12744            );
12745        }
12746        self.apply_computed_overlay_format_effect(
12747            chunk.sheet_id,
12748            chunk.col0,
12749            chunk.chunk_idx,
12750            chunk.format_effect,
12751        );
12752    }
12753
12754    fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
12755        if chunk.entries.is_empty() {
12756            return;
12757        }
12758        let start = chunk.entries[0].row_in_chunk;
12759        let values: Vec<OverlayValue> =
12760            chunk.entries.into_iter().map(|entry| entry.value).collect();
12761        if let Some(fragment) = OverlayFragment::run_range(start, values) {
12762            self.apply_computed_overlay_fragment(
12763                chunk.sheet_id,
12764                chunk.col0,
12765                chunk.chunk_idx,
12766                fragment,
12767            );
12768        }
12769        self.apply_computed_overlay_format_effect(
12770            chunk.sheet_id,
12771            chunk.col0,
12772            chunk.chunk_idx,
12773            chunk.format_effect,
12774        );
12775    }
12776
12777    fn apply_computed_overlay_format_effect(
12778        &mut self,
12779        sheet_id: SheetId,
12780        col0: u32,
12781        chunk_idx: usize,
12782        effect: ComputedWriteChunkFormatEffect,
12783    ) {
12784        if !(self.config.arrow_storage_enabled
12785            && self.config.delta_overlay_enabled
12786            && self.config.write_formula_overlay_enabled)
12787            || self.computed_overlay_mirroring_disabled
12788        {
12789            return;
12790        }
12791
12792        let sheet_name = self.graph.sheet_name(sheet_id);
12793        let Some(sheet) = self.arrow_sheets.sheet_mut(sheet_name) else {
12794            return;
12795        };
12796        let Some(chunk) = sheet
12797            .columns
12798            .get_mut(col0 as usize)
12799            .and_then(|column| column.chunk_mut(chunk_idx))
12800        else {
12801            return;
12802        };
12803        match effect {
12804            ComputedWriteChunkFormatEffect::NoFormatWork => {}
12805            ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Range {
12806                start,
12807                end,
12808            }) => {
12809                #[cfg(test)]
12810                {
12811                    self.computed_overlay_stale_clear_range_effects_for_test = self
12812                        .computed_overlay_stale_clear_range_effects_for_test
12813                        .saturating_add(1);
12814                }
12815                chunk.computed_overlay.clear_format_range(start, end);
12816            }
12817            ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Offsets(
12818                offsets,
12819            )) => {
12820                #[cfg(test)]
12821                {
12822                    self.computed_overlay_stale_clear_offset_attempts_for_test = self
12823                        .computed_overlay_stale_clear_offset_attempts_for_test
12824                        .saturating_add(offsets.len() as u64);
12825                }
12826                chunk.computed_overlay.clear_format_offsets(&offsets);
12827            }
12828            ComputedWriteChunkFormatEffect::SetExplicit(formats) => {
12829                #[cfg(test)]
12830                {
12831                    self.computed_overlay_set_explicit_entry_operations_for_test = self
12832                        .computed_overlay_set_explicit_entry_operations_for_test
12833                        .saturating_add(formats.len() as u64);
12834                }
12835                for (row_in_chunk, format_id) in formats {
12836                    chunk.computed_overlay.set_format(row_in_chunk, format_id);
12837                }
12838            }
12839        }
12840    }
12841
12842    fn apply_computed_overlay_fragment(
12843        &mut self,
12844        sheet_id: SheetId,
12845        col0: u32,
12846        chunk_idx: usize,
12847        fragment: OverlayFragment,
12848    ) {
12849        if !(self.config.arrow_storage_enabled
12850            && self.config.delta_overlay_enabled
12851            && self.config.write_formula_overlay_enabled)
12852        {
12853            return;
12854        }
12855        if self.computed_overlay_mirroring_disabled {
12856            return;
12857        }
12858
12859        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
12860        self.ensure_arrow_sheet(&sheet_name);
12861
12862        let col0 = col0 as usize;
12863        let asheet = self
12864            .arrow_sheets
12865            .sheet_mut(&sheet_name)
12866            .expect("ArrowSheet must exist");
12867
12868        let cur_cols = asheet.columns.len();
12869        if col0 >= cur_cols {
12870            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12871        }
12872
12873        let start_row0 = asheet
12874            .chunk_starts
12875            .get(chunk_idx)
12876            .copied()
12877            .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
12878        let required_rows =
12879            start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
12880        if required_rows > asheet.nrows as usize {
12881            if asheet.columns.is_empty() {
12882                asheet.insert_columns(0, 1);
12883            }
12884            asheet.ensure_row_capacity(required_rows);
12885        }
12886
12887        let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
12888            return;
12889        };
12890        let delta = ch.computed_overlay.apply_fragment(fragment);
12891        self.adjust_computed_overlay_bytes(delta);
12892
12893        if let Some(cap) = self.config.max_overlay_memory_bytes
12894            && self.computed_overlay_bytes_estimate > cap
12895        {
12896            self.disable_computed_overlay_mirroring_due_to_budget(cap);
12897        }
12898    }
12899
12900    #[inline]
12901    fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
12902        if delta >= 0 {
12903            self.computed_overlay_bytes_estimate = self
12904                .computed_overlay_bytes_estimate
12905                .saturating_add(delta as usize);
12906        } else {
12907            self.computed_overlay_bytes_estimate = self
12908                .computed_overlay_bytes_estimate
12909                .saturating_sub((-delta) as usize);
12910        }
12911    }
12912
12913    fn clear_all_computed_overlays(&mut self) {
12914        let mut freed_total = 0usize;
12915        for sh in self.arrow_sheets.sheets.iter_mut() {
12916            for col in sh.columns.iter_mut() {
12917                for ch in col.chunks.iter_mut() {
12918                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
12919                }
12920                for ch in col.sparse_chunks.values_mut() {
12921                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
12922                }
12923            }
12924        }
12925        self.computed_overlay_bytes_estimate = self
12926            .computed_overlay_bytes_estimate
12927            .saturating_sub(freed_total);
12928    }
12929
12930    fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
12931        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
12932        // Handle budget pressure by compacting computed overlays into base lanes.
12933        self.compact_all_computed_overlays();
12934    }
12935
12936    /// Fold all computed overlay entries across all sheets into their base arrays.
12937    /// This preserves data while freeing overlay memory, allowing mirroring to continue.
12938    fn compact_all_computed_overlays(&mut self) {
12939        let mut freed_total = 0usize;
12940        for sheet in self.arrow_sheets.sheets.iter_mut() {
12941            for col_idx in 0..sheet.columns.len() {
12942                // Dense chunks
12943                let num_dense = sheet.columns[col_idx].chunks.len();
12944                for ch_idx in 0..num_dense {
12945                    freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
12946                }
12947                // Sparse chunks
12948                let sparse_keys: Vec<usize> = sheet.columns[col_idx]
12949                    .sparse_chunks
12950                    .keys()
12951                    .copied()
12952                    .collect();
12953                for ch_idx in sparse_keys {
12954                    freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
12955                }
12956            }
12957        }
12958        self.computed_overlay_bytes_estimate = self
12959            .computed_overlay_bytes_estimate
12960            .saturating_sub(freed_total);
12961        self.overlay_compactions = self.overlay_compactions.saturating_add(1);
12962    }
12963
12964    fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
12965        let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
12966    }
12967
12968    fn record_vertex_value_to_overlay(
12969        &mut self,
12970        vertex_id: VertexId,
12971        value: &LiteralValue,
12972        computed_writes: Option<&mut ComputedWriteBuffer>,
12973    ) -> Result<(), ExcelError> {
12974        if !(self.config.arrow_storage_enabled
12975            && self.config.delta_overlay_enabled
12976            && self.config.write_formula_overlay_enabled)
12977        {
12978            return Ok(());
12979        }
12980        if self.computed_overlay_mirroring_disabled {
12981            return Ok(());
12982        }
12983        if !matches!(
12984            self.graph.get_vertex_kind(vertex_id),
12985            VertexKind::FormulaScalar | VertexKind::FormulaArray
12986        ) {
12987            return Ok(());
12988        }
12989        let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
12990            return Ok(());
12991        };
12992        let Some(buffer) = computed_writes else {
12993            // Unbuffered: one sheet lookup for the value and its format.
12994            let format_id = self.derived_formats.get(&cell);
12995            self.write_computed_cell_0based(cell, value, format_id);
12996            return Ok(());
12997        };
12998        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
12999        let date_system = self.arrow_sheet_date_system(&sheet_name);
13000        let ov = Self::literal_to_overlay_value(value, date_system);
13001        {
13002            let format_id = self.derived_formats.get(&cell);
13003            buffer.push_cell_with_format(
13004                cell.sheet_id,
13005                cell.coord.row(),
13006                cell.coord.col(),
13007                ov,
13008                format_id,
13009            );
13010            if self.should_flush_computed_write_buffer(buffer) {
13011                self.flush_computed_write_buffer(buffer)?;
13012            }
13013        }
13014        Ok(())
13015    }
13016
13017    #[inline]
13018    fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
13019        self.config.max_overlay_memory_bytes.is_some_and(|cap| {
13020            if cap == 0 {
13021                return false;
13022            }
13023            self.computed_overlay_bytes_estimate
13024                .saturating_add(buffer.estimated_bytes())
13025                > cap
13026        })
13027    }
13028
13029    /// Estimated memory usage for computed overlays (formula/spill mirroring).
13030    pub fn overlay_memory_usage(&self) -> usize {
13031        self.computed_overlay_bytes_estimate
13032    }
13033
13034    #[cfg(test)]
13035    pub(crate) fn debug_overlay_compactions(&self) -> u64 {
13036        self.overlay_compactions
13037    }
13038
13039    #[cfg(test)]
13040    pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
13041        let mut total = 0usize;
13042        for sheet in &self.arrow_sheets.sheets {
13043            for column in &sheet.columns {
13044                for chunk in &column.chunks {
13045                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13046                }
13047                for chunk in column.sparse_chunks.values() {
13048                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13049                }
13050            }
13051        }
13052        self.computed_overlay_bytes_estimate = total;
13053        total
13054    }
13055
13056    fn resolve_sheet_locator_for_write(
13057        &mut self,
13058        loc: formualizer_common::SheetLocator<'_>,
13059        current_sheet: &str,
13060    ) -> Result<SheetId, ExcelError> {
13061        Ok(match loc {
13062            formualizer_common::SheetLocator::Id(id) => id,
13063            formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
13064            formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
13065        })
13066    }
13067
13068    fn resolve_sheet_locator_for_read(
13069        &self,
13070        loc: formualizer_common::SheetLocator<'_>,
13071        current_sheet: &str,
13072    ) -> Result<SheetId, ExcelError> {
13073        match loc {
13074            formualizer_common::SheetLocator::Id(id) => Ok(id),
13075            formualizer_common::SheetLocator::Name(name) => self
13076                .graph
13077                .sheet_id(name.as_ref())
13078                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13079            formualizer_common::SheetLocator::Current => self
13080                .graph
13081                .sheet_id(current_sheet)
13082                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13083        }
13084    }
13085
13086    /// Set a cell value
13087    pub fn set_cell_value(
13088        &mut self,
13089        sheet: &str,
13090        row: u32,
13091        col: u32,
13092        value: LiteralValue,
13093    ) -> Result<(), ExcelError> {
13094        self.observe_function_semantic_epoch()?;
13095        let sheet_existed = self.graph.sheet_id(sheet).is_some();
13096        let sheet_id = self.graph.sheet_id_mut(sheet);
13097        let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13098        let replaced = self
13099            .graph
13100            .get_vertex_id_for_address(&cell_ref)
13101            .filter(|&vertex| {
13102                matches!(
13103                    self.graph.get_vertex_kind(vertex),
13104                    VertexKind::FormulaScalar | VertexKind::FormulaArray
13105                )
13106            });
13107        let replaced_formula = replaced.is_some();
13108        // A value replacing a spilled anchor takes its spill with it (no
13109        // evaluation will revisit the anchor); anchors that spill blocked
13110        // wake through the cleared region.
13111        if let Some(anchor) = replaced
13112            && self.graph.spill_registry_has_anchor(anchor)
13113        {
13114            self.clear_spill_projection_and_mirror(anchor, None);
13115        }
13116        self.graph.set_cell_value(sheet, row, col, value.clone())?;
13117        self.clear_cell_format_state(sheet, cell_ref);
13118        self.record_changed_cell(sheet, row, col);
13119        if !sheet_existed || replaced_formula {
13120            self.mark_topology_edited();
13121        }
13122        // Mirror into Arrow overlay when enabled
13123        self.mirror_value_to_overlay(sheet, row, col, &value);
13124        // Advance snapshot to reflect external mutation.
13125        self.mark_data_edited();
13126        Ok(())
13127    }
13128
13129    /// Record a single-cell change: invalidates pending spills blocked on it.
13130    fn record_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
13131        let sheet_id = self.graph.sheet_id_mut(sheet);
13132        self.record_structural_change(StructuralScope::Cell {
13133            sheet: sheet_id,
13134            row: row.saturating_sub(1),
13135            col: col.saturating_sub(1),
13136        });
13137    }
13138
13139    fn record_change_for_event(&mut self, event: &ChangeEvent) {
13140        match event {
13141            ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
13142                self.record_structural_change(StructuralScope::Cell {
13143                    sheet: addr.sheet_id,
13144                    row: addr.coord.row(),
13145                    col: addr.coord.col(),
13146                });
13147            }
13148            ChangeEvent::SpillCommitted { new, .. } => {
13149                if let Some(scope) = Self::structural_scope_from_cells(&new.target_cells) {
13150                    self.record_structural_change(scope);
13151                }
13152            }
13153            ChangeEvent::SpillCleared { old, .. } => {
13154                if let Some(scope) = Self::structural_scope_from_cells(&old.target_cells) {
13155                    self.record_structural_change(scope);
13156                }
13157            }
13158            ChangeEvent::DefineName { .. }
13159            | ChangeEvent::UpdateName { .. }
13160            | ChangeEvent::DeleteName { .. }
13161            | ChangeEvent::NamedRangeAdjusted { .. } => {
13162                // Direct name events are preflighted by the logged-name APIs.
13163                // Structural entry points preflight spans before emitting a
13164                // NamedRangeAdjusted event. Epoch changes only rebuild caches.
13165                self.record_structural_change(StructuralScope::AllSheets);
13166            }
13167            ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
13168                // Structural entry points publish their axis delta once after
13169                // the graph and Arrow commits.
13170            }
13171            ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
13172                self.record_structural_change(StructuralScope::Region(Region::whole_row(
13173                    *sheet_id, *row0,
13174                )));
13175            }
13176            ChangeEvent::AddVertex { .. }
13177            | ChangeEvent::RemoveVertex { .. }
13178            | ChangeEvent::EdgeAdded { .. }
13179            | ChangeEvent::EdgeRemoved { .. }
13180            | ChangeEvent::CompoundStart { .. }
13181            | ChangeEvent::CompoundEnd { .. }
13182            | ChangeEvent::StagedFormulaCellChanged { .. } => {}
13183        }
13184    }
13185
13186    fn record_structural_change(&mut self, scope: StructuralScope) {
13187        if let StructuralScope::Cell { sheet, row, col } = scope {
13188            self.wake_spill_owner_of_entered_formula(sheet, row, col);
13189        }
13190        self.wake_spill_blocker_waiters(scope);
13191        self.invalidate_pending_spills(scope);
13192    }
13193
13194    /// Wake anchors waiting on a blocking cell inside `scope`, with their
13195    /// dependents: the anchor's value changes when it spills. A woken anchor
13196    /// registers again if it is still blocked.
13197    fn wake_spill_blocker_waiters(&mut self, scope: StructuralScope) {
13198        if self.spill_blocker_waiters.is_empty() {
13199            return;
13200        }
13201        let woken = self.spill_blocker_waiters.take_affected(scope);
13202        let woken: Vec<VertexId> = woken
13203            .into_iter()
13204            .filter(|&vertex| {
13205                self.graph.vertex_exists(vertex)
13206                    && matches!(
13207                        self.graph.get_vertex_kind(vertex),
13208                        VertexKind::FormulaScalar | VertexKind::FormulaArray
13209                    )
13210            })
13211            .collect();
13212        if !woken.is_empty() {
13213            self.graph.mark_dirty_many(&woken);
13214        }
13215    }
13216
13217    /// A formula entered into a live spill blocks it: wake the owning anchor
13218    /// so its next recalc re-plans (and reports `#SPILL!`). The spill keeps
13219    /// its registry until then; clearing it leaves the formula cell alone.
13220    fn wake_spill_owner_of_entered_formula(&mut self, sheet: SheetId, row: u32, col: u32) {
13221        let cell = self.graph.make_cell_ref_internal(sheet, row, col);
13222        if let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
13223            && self.graph.is_foreign_formula_cell(&cell, owner)
13224        {
13225            self.graph.mark_dirty_many(&[owner]);
13226        }
13227    }
13228
13229    fn structural_scope_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
13230        let first = cells.first()?;
13231        let sheet_id = first.sheet_id;
13232        if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
13233            return Some(StructuralScope::OpaqueGlobal);
13234        }
13235        let mut row_start = first.coord.row();
13236        let mut row_end = row_start;
13237        let mut col_start = first.coord.col();
13238        let mut col_end = col_start;
13239        for cell in cells.iter().skip(1) {
13240            row_start = row_start.min(cell.coord.row());
13241            row_end = row_end.max(cell.coord.row());
13242            col_start = col_start.min(cell.coord.col());
13243            col_end = col_end.max(cell.coord.col());
13244        }
13245        Some(StructuralScope::Region(Region::rect(
13246            sheet_id, row_start, row_end, col_start, col_end,
13247        )))
13248    }
13249
13250    pub fn set_cell_value_ref(
13251        &mut self,
13252        cell: formualizer_common::SheetCellRef<'_>,
13253        current_sheet: &str,
13254        value: LiteralValue,
13255    ) -> Result<(), ExcelError> {
13256        let owned = cell.into_owned();
13257        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13258        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13259        self.set_cell_value(
13260            &sheet_name,
13261            owned.coord.row() + 1,
13262            owned.coord.col() + 1,
13263            value,
13264        )
13265    }
13266
13267    pub fn set_cell_formula_ref(
13268        &mut self,
13269        cell: formualizer_common::SheetCellRef<'_>,
13270        current_sheet: &str,
13271        ast: ASTNode,
13272    ) -> Result<(), ExcelError> {
13273        let owned = cell.into_owned();
13274        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13275        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13276        self.set_cell_formula(
13277            &sheet_name,
13278            owned.coord.row() + 1,
13279            owned.coord.col() + 1,
13280            ast,
13281        )
13282    }
13283
13284    pub fn get_cell_value_ref(
13285        &self,
13286        cell: formualizer_common::SheetCellRef<'_>,
13287        current_sheet: &str,
13288    ) -> Result<Option<LiteralValue>, ExcelError> {
13289        let owned = cell.into_owned();
13290        let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
13291        let sheet_name = self.graph.sheet_name(sheet_id);
13292        Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
13293    }
13294
13295    pub fn resolve_range_view_sheet_ref<'c>(
13296        &'c self,
13297        r: &formualizer_common::SheetRef<'_>,
13298        current_sheet: &str,
13299    ) -> Result<RangeView<'c>, ExcelError> {
13300        use formualizer_common::SheetLocator;
13301
13302        let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
13303            match loc {
13304                SheetLocator::Current => Ok(None),
13305                SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
13306                SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
13307            }
13308        };
13309
13310        let rt = match r {
13311            formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
13312                sheet: sheet_to_opt_name(cell.sheet.clone())?,
13313                row: cell.coord.row() + 1,
13314                col: cell.coord.col() + 1,
13315                row_abs: cell.coord.row_abs(),
13316                col_abs: cell.coord.col_abs(),
13317            },
13318            formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
13319                sheet: sheet_to_opt_name(range.sheet.clone())?,
13320                start_row: range.start_row.map(|b| b.index + 1),
13321                start_col: range.start_col.map(|b| b.index + 1),
13322                end_row: range.end_row.map(|b| b.index + 1),
13323                end_col: range.end_col.map(|b| b.index + 1),
13324                start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
13325                start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
13326                end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
13327                end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
13328            },
13329        };
13330
13331        crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
13332    }
13333
13334    /// Set a cell formula
13335    pub fn set_cell_formula(
13336        &mut self,
13337        sheet: &str,
13338        row: u32,
13339        col: u32,
13340        ast: ASTNode,
13341    ) -> Result<(), ExcelError> {
13342        self.observe_function_semantic_epoch()?;
13343        let sheet_id = self.graph.sheet_id_mut(sheet);
13344        let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13345        let ingested = {
13346            let mut pipeline = self.ingest_pipeline();
13347            pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
13348        };
13349        self.graph.set_cell_formula_with_plan(
13350            sheet,
13351            row,
13352            col,
13353            ingested.ast_id,
13354            &ingested.dep_plan,
13355            ingested.dep_plan.volatile,
13356            ingested.dep_plan.dynamic,
13357        )?;
13358        self.clear_cell_format_state(sheet, placement);
13359        self.record_changed_cell(sheet, row, col);
13360
13361        // If the cell previously held a user value in the delta overlay, it must not continue
13362        // to mask the formula result under Arrow-canonical reads (overlay precedence is
13363        // delta -> computed -> base). Remove the overlay entry instead of writing `Empty`,
13364        // because an explicit `Empty` overlay would still take precedence over computed values.
13365        self.clear_delta_overlay_cell(sheet, row, col);
13366
13367        // Advance snapshot to reflect external mutation
13368        self.mark_topology_edited();
13369        Ok(())
13370    }
13371
13372    /// Bulk set many formulas on a sheet. Skips per-cell snapshot bumping and minimizes edge rebuilds.
13373    pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
13374    where
13375        I: IntoIterator<Item = (u32, u32, ASTNode)>,
13376    {
13377        let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
13378        let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
13379        let sheet_id = self.graph.sheet_id_mut(sheet);
13380        let ingested = {
13381            let mut pipeline = self.ingest_pipeline();
13382            let inputs = collected.into_iter().map(|(row, col, ast)| {
13383                let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13384                (FormulaAstInput::Tree(ast), placement, None)
13385            });
13386            pipeline.ingest_batch(inputs)?
13387        };
13388        let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
13389            .into_iter()
13390            .map(|formula| {
13391                (
13392                    formula.placement.coord.row() + 1,
13393                    formula.placement.coord.col() + 1,
13394                    formula.ast_id,
13395                    formula.dep_plan,
13396                )
13397            })
13398            .collect();
13399        let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
13400        for (row, col) in edited_cells {
13401            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13402            self.clear_cell_format_state(sheet, cell);
13403            self.record_changed_cell(sheet, row, col);
13404        }
13405        // Single topology bump after batch
13406        if n > 0 {
13407            self.mark_topology_edited();
13408        }
13409        Ok(n)
13410    }
13411
13412    /// Declare the formula cell at `sheet!(row, col)` (1-based) a dynamic
13413    /// array anchor whose spill identity comes from the source document, for
13414    /// example an XLSX `t="array"` formula bound to XLDAPR cell metadata.
13415    ///
13416    /// This changes only spill references to the anchor (`A1#`,
13417    /// `_xlfn.ANCHORARRAY(A1)`). A declared anchor with a committed
13418    /// multi-cell spill resolves to that spill, exactly as an undeclared
13419    /// one. Without a committed spill, a declared anchor whose current
13420    /// result is a scalar (not an error, so not a blocked `#SPILL!`
13421    /// anchor, and not empty) resolves to its own cell as a 1x1 reference,
13422    /// matching a spilled 1x1 dynamic array. Every other case keeps the
13423    /// `#REF!` of an undeclared anchor. Undeclared cells are unaffected, and
13424    /// fresh 1x1 results are not given spill identity by function name.
13425    ///
13426    /// The declaration belongs to the formula vertex. It is dropped when the
13427    /// cell's formula is replaced, the cell is cleared or overwritten with a
13428    /// value, the vertex is removed (including with its sheet), or a
13429    /// structural row/column edit moves the vertex. A structural edit that
13430    /// only adjusts the formula's references, without moving the anchor,
13431    /// keeps it. Declaring again re-establishes it.
13432    ///
13433    /// The anchor and its dependents are marked dirty, so the next
13434    /// evaluation observes the declaration.
13435    ///
13436    /// Returns `#REF!` if the sheet does not exist or the cell holds no
13437    /// formula vertex. A formula still staged by deferred graph building has
13438    /// no vertex: build the graph (for example [`Self::build_graph_all`])
13439    /// before declaring.
13440    pub fn declare_dynamic_array_anchor(
13441        &mut self,
13442        sheet: &str,
13443        row: u32,
13444        col: u32,
13445    ) -> Result<(), ExcelError> {
13446        let not_a_formula = || {
13447            ExcelError::new(ExcelErrorKind::Ref)
13448                .with_message("Declared dynamic array anchor must be a formula cell")
13449        };
13450        if row == 0 || col == 0 {
13451            return Err(not_a_formula());
13452        }
13453        if self.get_staged_formula_text(sheet, row, col).is_some() {
13454            return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
13455                "Declared dynamic array anchor is still staged; build the graph first",
13456            ));
13457        }
13458        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(not_a_formula)?;
13459        let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13460        let vertex = self
13461            .graph
13462            .cell_vertex_mut(&cell)
13463            .filter(|v| self.graph.vertex_has_formula(*v))
13464            .ok_or_else(not_a_formula)?;
13465        if self.graph.declare_dynamic_anchor(vertex, cell) {
13466            self.graph.mark_dirty_many(&[vertex]);
13467            self.mark_topology_edited();
13468        }
13469        Ok(())
13470    }
13471
13472    /// Declare a legacy array formula with a fixed output rectangle after
13473    /// ingestion and before evaluation. Coordinates are one-based. Replacement,
13474    /// removal or movement of the formula clears the declaration.
13475    /// Single-cell declarations do not register a spill anchor. Requires
13476    /// `EvalConfig::family_execution = false` to bypass family memoization.
13477    pub fn declare_fixed_array_formula(
13478        &mut self,
13479        sheet: &str,
13480        row: u32,
13481        col: u32,
13482        rows: u32,
13483        cols: u32,
13484    ) -> Result<(), ExcelError> {
13485        if self.config.family_execution {
13486            return Err(ExcelError::new(ExcelErrorKind::Ref)
13487                .with_message("Fixed arrays require family_execution = false"));
13488        }
13489        if row == 0
13490            || col == 0
13491            || rows == 0
13492            || cols == 0
13493            || row.checked_add(rows - 1).is_none_or(|r| r > 1_048_576)
13494            || col.checked_add(cols - 1).is_none_or(|c| c > 16_384)
13495            || u64::from(rows) * u64::from(cols) > u64::from(self.config.spill.max_spill_cells)
13496        {
13497            return Err(ExcelError::new(ExcelErrorKind::Ref)
13498                .with_message("Invalid or over-cap fixed array extent"));
13499        }
13500        if self.get_staged_formula_text(sheet, row, col).is_some() {
13501            return Err(ExcelError::new(ExcelErrorKind::Ref)
13502                .with_message("Fixed array formula is still staged; build the graph first"));
13503        }
13504        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
13505            ExcelError::new(ExcelErrorKind::Ref).with_message("Unknown fixed array sheet")
13506        })?;
13507        let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13508        let vertex = self
13509            .graph
13510            .cell_vertex_mut(&cell)
13511            .filter(|v| self.graph.vertex_has_formula(*v))
13512            .ok_or_else(|| {
13513                ExcelError::new(ExcelErrorKind::Ref)
13514                    .with_message("Fixed array anchor must be a formula cell")
13515            })?;
13516        if (rows == 1 && cols == 1 && self.graph.fixed_single_arrays.contains(&vertex))
13517            || self.graph.fixed_array_shapes.get(&vertex) == Some(&(rows, cols))
13518        {
13519            return Ok(());
13520        }
13521        if self.graph.fixed_single_arrays.contains(&vertex)
13522            || self.graph.spill_registry_has_anchor(vertex)
13523            || self.graph.is_current_declared_dynamic_anchor(vertex)
13524        {
13525            return Err(ExcelError::new(ExcelErrorKind::Ref)
13526                .with_message("Array declaration already exists; replace the formula first"));
13527        }
13528        if rows == 1 && cols == 1 {
13529            self.graph.fixed_single_arrays.insert(vertex);
13530        } else {
13531            let mut targets = Vec::with_capacity((rows * cols) as usize);
13532            for r in 0..rows {
13533                for c in 0..cols {
13534                    targets.push(self.graph.make_cell_ref(sheet, row + r, col + c));
13535                }
13536            }
13537            self.graph.plan_spill_region(vertex, &targets)?;
13538            // Reserve ownership before evaluation, even for Empty members.
13539            self.graph
13540                .commit_spill_region_atomic_with_fault(vertex, targets, vec![], None)?;
13541            self.graph.fixed_array_shapes.insert(vertex, (rows, cols));
13542        }
13543        self.graph.mark_dirty_many(&[vertex]);
13544        self.mark_topology_edited();
13545        Ok(())
13546    }
13547
13548    #[inline]
13549    fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
13550        match v {
13551            LiteralValue::Empty => None,
13552            LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
13553            other => Some(other),
13554        }
13555    }
13556
13557    fn materialize_temporal_egress(
13558        value: LiteralValue,
13559        class: Option<&formualizer_common::numfmt::FormatClass>,
13560        policy: crate::engine::TemporalEgress,
13561        date_system: crate::engine::DateSystem,
13562    ) -> LiteralValue {
13563        use formualizer_common::numfmt::FormatClass;
13564        if policy == crate::engine::TemporalEgress::Serial {
13565            return value;
13566        }
13567        let LiteralValue::Number(serial) = value else {
13568            return value;
13569        };
13570        match class {
13571            Some(FormatClass::Date) => {
13572                formualizer_common::try_serial_to_date_for(date_system, serial)
13573                    .map(LiteralValue::Date)
13574                    .unwrap_or(LiteralValue::Number(serial))
13575            }
13576            Some(FormatClass::DateTime) => {
13577                formualizer_common::try_serial_to_datetime_for(date_system, serial)
13578                    .map(LiteralValue::DateTime)
13579                    .unwrap_or(LiteralValue::Number(serial))
13580            }
13581            Some(FormatClass::Time) => {
13582                let seconds = (serial.rem_euclid(1.0) * 86_400.0).round() as u32 % 86_400;
13583                chrono::NaiveTime::from_num_seconds_from_midnight_opt(seconds, 0)
13584                    .map(LiteralValue::Time)
13585                    .unwrap_or(LiteralValue::Number(serial))
13586            }
13587            Some(FormatClass::Duration) => {
13588                let nanos = (serial * 86_400.0 * 1_000_000_000.0).round();
13589                if nanos.is_finite() && nanos >= i64::MIN as f64 && nanos <= i64::MAX as f64 {
13590                    LiteralValue::Duration(chrono::Duration::nanoseconds(nanos as i64))
13591                } else {
13592                    LiteralValue::Number(serial)
13593                }
13594            }
13595            _ => LiteralValue::Number(serial),
13596        }
13597    }
13598
13599    pub(crate) fn effective_format_id(
13600        &self,
13601        sheet: &str,
13602        row: u32,
13603        col: u32,
13604    ) -> Option<crate::format::FormatId> {
13605        let arrow = self.arrow_sheets.sheet(sheet).and_then(|arrow| {
13606            arrow.format_id(
13607                row.saturating_sub(1) as usize,
13608                col.saturating_sub(1) as usize,
13609            )
13610        });
13611        arrow.or_else(|| {
13612            let sheet_id = self.graph.sheet_id(sheet)?;
13613            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13614            self.derived_formats.get(&cell)
13615        })
13616    }
13617
13618    /// Get a cell value through the single temporal egress boundary.
13619    pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13620        let raw = self.read_cell_value(sheet, row, col)?;
13621        let format = self.effective_format_id(sheet, row, col);
13622        let class = format.and_then(|id| self.format_registry.class(id));
13623        Self::normalize_public_cell_read(Self::materialize_temporal_egress(
13624            raw,
13625            class,
13626            self.config.temporal_egress,
13627            self.config.date_system,
13628        ))
13629    }
13630
13631    /// Read a rectangular range through the temporal egress boundary.
13632    pub fn get_range_values(
13633        &self,
13634        sheet: &str,
13635        sr: u32,
13636        sc: u32,
13637        er: u32,
13638        ec: u32,
13639    ) -> Vec<Vec<LiteralValue>> {
13640        let height = er.saturating_sub(sr).saturating_add(1) as usize;
13641        let width = ec.saturating_sub(sc).saturating_add(1) as usize;
13642        let Some(asheet) = self.sheet_store().sheet(sheet) else {
13643            return vec![vec![LiteralValue::Empty; width]; height];
13644        };
13645        let view = asheet.range_view(
13646            sr.saturating_sub(1) as usize,
13647            sc.saturating_sub(1) as usize,
13648            er.saturating_sub(1) as usize,
13649            ec.saturating_sub(1) as usize,
13650        );
13651        let sheet_id = self.graph.sheet_id(sheet);
13652        let derived_formats = &self.derived_formats;
13653        let has_derived_formats =
13654            sheet_id.is_some_and(|sheet_id| derived_formats.any(|cell| cell.sheet_id == sheet_id));
13655        let mut out = Vec::with_capacity(height);
13656        if !asheet.has_formats() && !has_derived_formats {
13657            for rr in 0..height {
13658                let mut row = Vec::with_capacity(width);
13659                for cc in 0..width {
13660                    row.push(view.get_cell(rr, cc));
13661                }
13662                out.push(row);
13663            }
13664            return out;
13665        }
13666        let format_registry = &self.format_registry;
13667        for rr in 0..height {
13668            let mut row = Vec::with_capacity(width);
13669            for cc in 0..width {
13670                let raw = view.get_cell(rr, cc);
13671                let row0 = sr.saturating_sub(1).saturating_add(rr as u32);
13672                let col0 = sc.saturating_sub(1).saturating_add(cc as u32);
13673                let format = asheet.format_id(row0 as usize, col0 as usize).or_else(|| {
13674                    let cell = CellRef::new(sheet_id?, Coord::new(row0, col0, true, true));
13675                    derived_formats.get(&cell)
13676                });
13677                let class = format.and_then(|id| format_registry.class(id));
13678                row.push(Self::materialize_temporal_egress(
13679                    raw,
13680                    class,
13681                    self.config.temporal_egress,
13682                    self.config.date_system,
13683                ));
13684            }
13685            out.push(row);
13686        }
13687        out
13688    }
13689
13690    /// Unified internal read API for a single cell value (Arrow-truth).
13691    /// The 1x1 spill fallback of a declared dynamic-array anchor: its
13692    /// committed result is a non-error, non-empty scalar. Scheduling orders
13693    /// the anchor before its spill readers (the reference keeps a static
13694    /// edge to it), exactly as for committed multi-cell spills.
13695    fn declared_anchor_holds_scalar(&self, anchor: CellRef) -> bool {
13696        let sheet = self.graph.sheet_name(anchor.sheet_id);
13697        if sheet.is_empty() {
13698            return false;
13699        }
13700        matches!(
13701            self.read_cell_value(sheet, anchor.coord.row() + 1, anchor.coord.col() + 1),
13702            Some(value) if !matches!(
13703                value,
13704                LiteralValue::Error(_)
13705                    | LiteralValue::Array(_)
13706                    | LiteralValue::Pending
13707                    | LiteralValue::Empty
13708            )
13709        )
13710    }
13711
13712    pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13713        let asheet = self.sheet_store().sheet(sheet)?;
13714        let r0 = row.saturating_sub(1) as usize;
13715        let c0 = col.saturating_sub(1) as usize;
13716        let v = asheet.get_cell_value(r0, c0);
13717        if matches!(v, LiteralValue::Empty) {
13718            None
13719        } else {
13720            Some(v)
13721        }
13722    }
13723
13724    /// Unified internal read API for a range of cell values (Arrow-truth).
13725    pub(crate) fn read_range_values(
13726        &self,
13727        sheet: &str,
13728        sr: u32,
13729        sc: u32,
13730        er: u32,
13731        ec: u32,
13732    ) -> RangeView<'_> {
13733        let Some(asheet) = self.sheet_store().sheet(sheet) else {
13734            return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
13735        };
13736        if er < sr || ec < sc {
13737            return asheet.range_view(1, 1, 0, 0);
13738        }
13739        let sr0 = sr.saturating_sub(1) as usize;
13740        let sc0 = sc.saturating_sub(1) as usize;
13741        let er0 = er.saturating_sub(1) as usize;
13742        let ec0 = ec.saturating_sub(1) as usize;
13743        asheet.range_view(sr0, sc0, er0, ec0)
13744    }
13745
13746    /// Get formula AST (if any) and current stored value for a cell
13747    pub fn get_cell(
13748        &self,
13749        sheet: &str,
13750        row: u32,
13751        col: u32,
13752    ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
13753        let v = self.get_cell_value(sheet, row, col);
13754        let sheet_id = self.graph.sheet_id(sheet)?;
13755        let coord = Coord::from_excel(row, col, true, true);
13756        let cell = CellRef::new(sheet_id, coord);
13757        if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
13758            let ast = self.graph.get_formula(vid);
13759            Some((ast, v))
13760        } else if v.is_some() || self.graph.had_legacy_cell_vertex(&cell) {
13761            // A referenced or emptied value cell has no vertex (decision
13762            // 27), but it is a cell the graph knows, as it was when it had
13763            // one (the interactive formula edit routes on this).
13764            Some((None, v))
13765        } else {
13766            None
13767        }
13768    }
13769
13770    /// Begin batch operations - defer CSR rebuilds for better performance
13771    pub fn begin_batch(&mut self) {
13772        self.graph.begin_batch();
13773    }
13774
13775    /// End batch operations and trigger CSR rebuild
13776    pub fn end_batch(&mut self) {
13777        self.graph.end_batch();
13778    }
13779
13780    /// Begin a deferred-dirty scope for a multi-edit batch: while active,
13781    /// every edit's dirty propagation queues its sources instead of running
13782    /// a full BFS per edit, and the outermost `end_deferred_dirty` flushes
13783    /// the union with ONE multi-source propagation (O(component) instead of
13784    /// O(edits × component)). See `DependencyGraph::begin_deferred_dirty`.
13785    ///
13786    /// Callers MUST run `end_deferred_dirty` on every exit path, including
13787    /// error returns; evaluation entry points `debug_assert` no scope leaked.
13788    pub fn begin_deferred_dirty(&mut self) {
13789        self.graph.begin_deferred_dirty();
13790    }
13791
13792    /// End a deferred-dirty scope, flushing the queued propagation when the
13793    /// outermost scope closes. See `Engine::begin_deferred_dirty`.
13794    pub fn end_deferred_dirty(&mut self) {
13795        let _ = self.graph.end_deferred_dirty();
13796    }
13797
13798    /// Total vertices processed by dirty-propagation BFS loops since graph
13799    /// creation. Perf-shape observability only (cross-crate tests assert
13800    /// batched edits propagate O(component), not O(edits × component)).
13801    pub fn dirty_propagation_visits(&self) -> u64 {
13802        self.graph.dirty_propagation_visits()
13803    }
13804
13805    /// Evaluate a single vertex.
13806    /// This is the core of the sequential evaluation logic for Milestone 3.1.
13807    #[inline]
13808    fn record_cell_if_changed(
13809        delta: &mut DeltaCollector,
13810        cell: &CellRef,
13811        old: &LiteralValue,
13812        new: &LiteralValue,
13813    ) {
13814        if old != new {
13815            delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
13816        }
13817    }
13818
13819    pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
13820        self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
13821            engine.observe_function_semantic_epoch()?;
13822            // A direct request selects exactly one vertex, regardless of its formula kind.
13823            engine.resource_checkpoint(1)?;
13824            if !engine.graph.vertex_exists(vertex_id) {
13825                return engine.evaluate_vertex_impl(vertex_id, None);
13826            }
13827            let is_formula = matches!(
13828                engine.graph.get_vertex_kind(vertex_id),
13829                VertexKind::FormulaScalar | VertexKind::FormulaArray
13830            );
13831            if is_formula {
13832                engine.begin_evaluation_request();
13833                #[cfg(any(test, feature = "legacy_oracle"))]
13834                engine.graph.flush_pending_edge_deltas();
13835                let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
13836                    vertex_id,
13837                )];
13838                engine.evaluate_legacy_target_roots(&roots, None)?;
13839            }
13840            engine.evaluate_vertex_impl(vertex_id, None)
13841        })
13842    }
13843
13844    /// Same rejection publication for owned arrays and pre-admitted range views.
13845    fn publish_oversized_spill(
13846        &mut self,
13847        vertex_id: VertexId,
13848        error: ExcelError,
13849        mut delta: Option<&mut DeltaCollector>,
13850    ) -> LiteralValue {
13851        self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
13852        let anchor = self
13853            .graph
13854            .get_cell_ref(vertex_id)
13855            .expect("cell ref for vertex");
13856        let spill_val = LiteralValue::Error(error);
13857        if let Some(d) = delta {
13858            let old = self
13859                .read_cell_value(
13860                    self.graph.sheet_name(anchor.sheet_id),
13861                    anchor.coord.row() + 1,
13862                    anchor.coord.col() + 1,
13863                )
13864                .unwrap_or(LiteralValue::Empty);
13865            if old != spill_val {
13866                d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
13867            }
13868        }
13869        self.graph.update_vertex_value_ref(vertex_id, &spill_val);
13870        if self.config.arrow_storage_enabled
13871            && self.config.delta_overlay_enabled
13872            && self.config.write_formula_overlay_enabled
13873        {
13874            let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
13875            self.mirror_value_to_computed_overlay(
13876                &sheet_name,
13877                anchor.coord.row() + 1,
13878                anchor.coord.col() + 1,
13879                &spill_val,
13880            );
13881        }
13882        spill_val
13883    }
13884
13885    fn evaluate_vertex_impl(
13886        &mut self,
13887        vertex_id: VertexId,
13888        delta: Option<&mut DeltaCollector>,
13889    ) -> Result<LiteralValue, ExcelError> {
13890        // Preserve the direct evaluator's compatibility behavior for invalid IDs, literal cells,
13891        // names, and other non-formula vertices. Only formula publication needs the C1a final
13892        // deadline checkpoint and effects pipeline.
13893        if !self.graph.vertex_exists(vertex_id) {
13894            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
13895                .with_message(format!("Vertex not found: {vertex_id:?}")));
13896        }
13897        if (self.active_resource_ledger.is_some()
13898            || self.graph.fixed_array_shapes.contains_key(&vertex_id))
13899            && matches!(
13900                self.graph.get_vertex_kind(vertex_id),
13901                VertexKind::FormulaScalar | VertexKind::FormulaArray
13902            )
13903        {
13904            let value = self
13905                .evaluate_vertex_immutable(vertex_id)
13906                .unwrap_or_else(LiteralValue::Error);
13907            let effects = self.plan_vertex_effects(vertex_id, value.clone())?;
13908            // Do not publish the selected result until the outer request's deadline succeeds.
13909            self.resource_checkpoint(0)?;
13910            let mut delta = delta;
13911            for effect in &effects {
13912                self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
13913            }
13914            return Ok(value);
13915        }
13916
13917        let mut delta = delta;
13918
13919        // Get vertex kind and check if it needs evaluation
13920        let kind = self.graph.get_vertex_kind(vertex_id);
13921        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
13922
13923        let view = match kind {
13924            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
13925                if let Some(view) = self.graph.formula_view(vertex_id) {
13926                    view
13927                } else {
13928                    return Ok(LiteralValue::Number(0.0));
13929                }
13930            }
13931            VertexKind::Empty | VertexKind::Cell => {
13932                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
13933                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
13934                    let row = cell_ref.coord.row() + 1;
13935                    let col = cell_ref.coord.col() + 1;
13936                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
13937                        return Ok(v);
13938                    }
13939                }
13940                return Ok(LiteralValue::Number(0.0));
13941            }
13942            VertexKind::NamedScalar => {
13943                let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
13944                return Ok(value);
13945            }
13946            VertexKind::NamedArray => {
13947                let value = self.evaluate_named_array(vertex_id, sheet_id)?;
13948                return Ok(value);
13949            }
13950            VertexKind::InfiniteRange
13951            | VertexKind::Range
13952            | VertexKind::External
13953            | VertexKind::Table => {
13954                // Not directly evaluatable here.
13955                return Ok(LiteralValue::Number(0.0));
13956            }
13957        };
13958
13959        // The interpreter uses a reference to the engine as the context.
13960        let sheet_name = self.graph.sheet_name(sheet_id);
13961        let cell_ref = self
13962            .graph
13963            .get_cell_ref(vertex_id)
13964            .expect("cell ref for vertex");
13965        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
13966
13967        let result = interpreter.evaluate_formula_view(
13968            view,
13969            self.graph.data_store(),
13970            self.graph.sheet_reg(),
13971        );
13972
13973        // If array result, perform spill from the anchor cell
13974        match result {
13975            Ok(cv) => {
13976                let derived_format = cv.format_id();
13977                self.record_derived_format(vertex_id, derived_format);
13978                let oversized_range = if matches!(cv, crate::traits::CalcValue::Range(_))
13979                    && self.graph.fixed_single_arrays.contains(&vertex_id)
13980                {
13981                    None
13982                } else {
13983                    crate::engine::result_finalization::range_spill_error(
13984                        &cv,
13985                        self.config.spill.max_spill_cells,
13986                    )
13987                };
13988                let is_oversized_range = oversized_range.is_some();
13989                let result_literal = if let Some(error) = oversized_range {
13990                    drop(cv);
13991                    LiteralValue::Error(error)
13992                } else {
13993                    self.materialize_formula_result(vertex_id, cv)
13994                };
13995                let output_sheet_name = sheet_name.to_string();
13996                self.write_computed_overlay_format_0based(
13997                    &output_sheet_name,
13998                    cell_ref.coord.row(),
13999                    cell_ref.coord.col(),
14000                    derived_format,
14001                );
14002                if is_oversized_range {
14003                    let LiteralValue::Error(error) = result_literal else {
14004                        unreachable!()
14005                    };
14006                    self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
14007                    return Ok(self.publish_oversized_spill(
14008                        vertex_id,
14009                        error,
14010                        delta.as_deref_mut(),
14011                    ));
14012                }
14013                match result_literal {
14014                    LiteralValue::Array(rows) => {
14015                        // Update kind to FormulaArray for tracking
14016                        self.graph
14017                            .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
14018                        // Build target cells rectangle starting from anchor
14019                        let anchor = self
14020                            .graph
14021                            .get_cell_ref(vertex_id)
14022                            .expect("cell ref for vertex");
14023                        let sheet_id = anchor.sheet_id;
14024                        let h = rows.len() as u32;
14025                        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
14026
14027                        // Hard cap to avoid vertex explosion from huge dynamic arrays.
14028                        let spill_cells = (h as u64).saturating_mul(w as u64);
14029                        if spill_cells > self.config.spill.max_spill_cells as u64 {
14030                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14031                                .with_message("SpillTooLarge")
14032                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14033                                    expected_rows: h,
14034                                    expected_cols: w,
14035                                });
14036                            return Ok(self.publish_oversized_spill(
14037                                vertex_id,
14038                                spill_err,
14039                                delta.as_deref_mut(),
14040                            ));
14041                        }
14042                        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity)
14043                        const PACKED_MAX_ROW: u32 = 1_048_575; // 20-bit max
14044                        const PACKED_MAX_COL: u32 = 16_383; // 14-bit max
14045                        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
14046                        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
14047                        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
14048                            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14049                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14050                                .with_message("Spill exceeds sheet bounds")
14051                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14052                                    expected_rows: h,
14053                                    expected_cols: w,
14054                                });
14055                            let spill_val = LiteralValue::Error(spill_err.clone());
14056                            if let Some(d) = delta.as_deref_mut() {
14057                                let old = self
14058                                    .read_cell_value(
14059                                        self.graph.sheet_name(anchor.sheet_id),
14060                                        anchor.coord.row() + 1,
14061                                        anchor.coord.col() + 1,
14062                                    )
14063                                    .unwrap_or(LiteralValue::Empty);
14064                                if old != spill_val {
14065                                    d.record_cell(
14066                                        anchor.sheet_id,
14067                                        anchor.coord.row(),
14068                                        anchor.coord.col(),
14069                                    );
14070                                }
14071                            }
14072                            self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14073                            if self.config.arrow_storage_enabled
14074                                && self.config.delta_overlay_enabled
14075                                && self.config.write_formula_overlay_enabled
14076                            {
14077                                let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14078                                self.mirror_value_to_computed_overlay(
14079                                    &sheet_name,
14080                                    anchor.coord.row() + 1,
14081                                    anchor.coord.col() + 1,
14082                                    &spill_val,
14083                                );
14084                            }
14085                            return Ok(spill_val);
14086                        }
14087                        let mut targets = Vec::new();
14088                        for r in 0..h {
14089                            for c in 0..w {
14090                                targets.push(self.graph.make_cell_ref_internal(
14091                                    sheet_id,
14092                                    anchor.coord.row() + r,
14093                                    anchor.coord.col() + c,
14094                                ));
14095                            }
14096                        }
14097
14098                        // Plan spill via spill manager shim
14099                        match self.spill_mgr.reserve(
14100                            vertex_id,
14101                            anchor,
14102                            SpillShape { rows: h, cols: w },
14103                            SpillMeta {
14104                                epoch: self.recalc_epoch,
14105                                config: self.config.spill,
14106                            },
14107                        ) {
14108                            Ok(()) => {
14109                                // Commit: write values to grid
14110                                // Default conflict policy is Error + FirstWins; reserve() enforces in-flight locks
14111                                // and plan_spill_region enforces overlap with committed formulas/spills/values.
14112                                if let Err(e) = self.commit_spill_and_mirror(
14113                                    vertex_id,
14114                                    &targets,
14115                                    rows.clone(),
14116                                    delta.as_deref_mut(),
14117                                ) {
14118                                    if e.kind != ExcelErrorKind::Spill {
14119                                        return Err(e);
14120                                    }
14121                                    // If commit fails, mark as error
14122                                    self.clear_spill_projection_and_mirror(
14123                                        vertex_id,
14124                                        delta.as_deref_mut(),
14125                                    );
14126                                    if let Some(d) = delta.as_deref_mut() {
14127                                        let old = self
14128                                            .read_cell_value(
14129                                                self.graph.sheet_name(anchor.sheet_id),
14130                                                anchor.coord.row() + 1,
14131                                                anchor.coord.col() + 1,
14132                                            )
14133                                            .unwrap_or(LiteralValue::Empty);
14134                                        let new = LiteralValue::Error(e.clone());
14135                                        if old != new {
14136                                            d.record_cell(
14137                                                anchor.sheet_id,
14138                                                anchor.coord.row(),
14139                                                anchor.coord.col(),
14140                                            );
14141                                        }
14142                                    }
14143                                    let err_val = LiteralValue::Error(e.clone());
14144                                    self.graph.update_vertex_value_ref(vertex_id, &err_val);
14145                                    if self.config.arrow_storage_enabled
14146                                        && self.config.delta_overlay_enabled
14147                                        && self.config.write_formula_overlay_enabled
14148                                    {
14149                                        let sheet_name =
14150                                            self.graph.sheet_name(anchor.sheet_id).to_string();
14151                                        self.mirror_value_to_computed_overlay(
14152                                            &sheet_name,
14153                                            anchor.coord.row() + 1,
14154                                            anchor.coord.col() + 1,
14155                                            &err_val,
14156                                        );
14157                                    }
14158                                    return Ok(err_val);
14159                                }
14160                                // Anchor shows the top-left value, like Excel
14161                                let top_left = rows
14162                                    .first()
14163                                    .and_then(|r| r.first())
14164                                    .cloned()
14165                                    .unwrap_or(LiteralValue::Empty);
14166                                self.graph.update_vertex_value_ref(vertex_id, &top_left);
14167                                Ok(top_left)
14168                            }
14169                            Err(e) => {
14170                                self.clear_spill_projection_and_mirror(
14171                                    vertex_id,
14172                                    delta.as_deref_mut(),
14173                                );
14174                                let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14175                                    .with_message(
14176                                        e.message.unwrap_or_else(|| "Spill blocked".to_string()),
14177                                    )
14178                                    .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14179                                        expected_rows: h,
14180                                        expected_cols: w,
14181                                    });
14182                                let spill_val = LiteralValue::Error(spill_err.clone());
14183                                if let Some(d) = delta.as_deref_mut() {
14184                                    let old = self
14185                                        .read_cell_value(
14186                                            self.graph.sheet_name(anchor.sheet_id),
14187                                            anchor.coord.row() + 1,
14188                                            anchor.coord.col() + 1,
14189                                        )
14190                                        .unwrap_or(LiteralValue::Empty);
14191                                    if old != spill_val {
14192                                        d.record_cell(
14193                                            anchor.sheet_id,
14194                                            anchor.coord.row(),
14195                                            anchor.coord.col(),
14196                                        );
14197                                    }
14198                                }
14199                                self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14200                                if self.config.arrow_storage_enabled
14201                                    && self.config.delta_overlay_enabled
14202                                    && self.config.write_formula_overlay_enabled
14203                                {
14204                                    let sheet_name =
14205                                        self.graph.sheet_name(anchor.sheet_id).to_string();
14206                                    self.mirror_value_to_computed_overlay(
14207                                        &sheet_name,
14208                                        anchor.coord.row() + 1,
14209                                        anchor.coord.col() + 1,
14210                                        &spill_val,
14211                                    );
14212                                }
14213                                Ok(spill_val)
14214                            }
14215                        }
14216                    }
14217                    other => {
14218                        // Scalar result: store value and ensure any previous spill is cleared
14219                        let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14220                        if let Some(d) = delta.as_deref_mut()
14221                            && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14222                        {
14223                            if spill_cells.is_empty() {
14224                                let old = self
14225                                    .read_cell_value(
14226                                        self.graph.sheet_name(anchor.sheet_id),
14227                                        anchor.coord.row() + 1,
14228                                        anchor.coord.col() + 1,
14229                                    )
14230                                    .unwrap_or(LiteralValue::Empty);
14231                                if old != other {
14232                                    d.record_cell(
14233                                        anchor.sheet_id,
14234                                        anchor.coord.row(),
14235                                        anchor.coord.col(),
14236                                    );
14237                                }
14238                            } else {
14239                                for cell in spill_cells.iter() {
14240                                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
14241                                    let old = self
14242                                        .get_cell_value(
14243                                            sheet_name,
14244                                            cell.coord.row() + 1,
14245                                            cell.coord.col() + 1,
14246                                        )
14247                                        .unwrap_or(LiteralValue::Empty);
14248                                    let new = if cell.sheet_id == anchor.sheet_id
14249                                        && cell.coord.row() == anchor.coord.row()
14250                                        && cell.coord.col() == anchor.coord.col()
14251                                    {
14252                                        other.clone()
14253                                    } else {
14254                                        LiteralValue::Empty
14255                                    };
14256                                    Self::record_cell_if_changed(d, cell, &old, &new);
14257                                }
14258                            }
14259                        }
14260                        self.graph.clear_spill_region(vertex_id);
14261                        if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14262                            self.record_structural_change(scope);
14263                        }
14264                        if self.config.arrow_storage_enabled
14265                            && self.config.delta_overlay_enabled
14266                            && self.config.write_formula_overlay_enabled
14267                        {
14268                            let empty = LiteralValue::Empty;
14269                            for cell in spill_cells.iter() {
14270                                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14271                                self.mirror_value_to_computed_overlay(
14272                                    &sheet_name,
14273                                    cell.coord.row() + 1,
14274                                    cell.coord.col() + 1,
14275                                    &empty,
14276                                );
14277                            }
14278                        }
14279                        self.graph.update_vertex_value_ref(vertex_id, &other);
14280                        // Optionally mirror into Arrow overlay for Arrow-backed reads
14281                        if self.config.arrow_storage_enabled
14282                            && self.config.delta_overlay_enabled
14283                            && self.config.write_formula_overlay_enabled
14284                        {
14285                            let anchor = self
14286                                .graph
14287                                .get_cell_ref(vertex_id)
14288                                .expect("cell ref for vertex");
14289                            let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14290                            self.mirror_value_to_computed_overlay(
14291                                &sheet_name,
14292                                anchor.coord.row() + 1,
14293                                anchor.coord.col() + 1,
14294                                &other,
14295                            );
14296                        }
14297                        Ok(other)
14298                    }
14299                }
14300            }
14301            Err(e) => {
14302                // Runtime Excel error: store as a cell value instead of propagating
14303                // as an exception so bulk eval paths don't fail the whole pass.
14304                let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14305                let err_val = LiteralValue::Error(e.clone());
14306                if let Some(d) = delta
14307                    && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14308                {
14309                    if spill_cells.is_empty() {
14310                        let old = self
14311                            .read_cell_value(
14312                                self.graph.sheet_name(anchor.sheet_id),
14313                                anchor.coord.row() + 1,
14314                                anchor.coord.col() + 1,
14315                            )
14316                            .unwrap_or(LiteralValue::Empty);
14317                        if old != err_val {
14318                            d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14319                        }
14320                    } else {
14321                        for cell in spill_cells.iter() {
14322                            let sheet_name = self.graph.sheet_name(cell.sheet_id);
14323                            let old = self
14324                                .get_cell_value(
14325                                    sheet_name,
14326                                    cell.coord.row() + 1,
14327                                    cell.coord.col() + 1,
14328                                )
14329                                .unwrap_or(LiteralValue::Empty);
14330                            let new = if cell.sheet_id == anchor.sheet_id
14331                                && cell.coord.row() == anchor.coord.row()
14332                                && cell.coord.col() == anchor.coord.col()
14333                            {
14334                                err_val.clone()
14335                            } else {
14336                                LiteralValue::Empty
14337                            };
14338                            Self::record_cell_if_changed(d, cell, &old, &new);
14339                        }
14340                    }
14341                }
14342                self.graph.clear_spill_region(vertex_id);
14343                if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14344                    self.record_structural_change(scope);
14345                }
14346                if self.config.arrow_storage_enabled
14347                    && self.config.delta_overlay_enabled
14348                    && self.config.write_formula_overlay_enabled
14349                {
14350                    let empty = LiteralValue::Empty;
14351                    for cell in spill_cells.iter() {
14352                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14353                        self.mirror_value_to_computed_overlay(
14354                            &sheet_name,
14355                            cell.coord.row() + 1,
14356                            cell.coord.col() + 1,
14357                            &empty,
14358                        );
14359                    }
14360                }
14361                self.graph.update_vertex_value_ref(vertex_id, &err_val);
14362                if self.config.arrow_storage_enabled
14363                    && self.config.delta_overlay_enabled
14364                    && self.config.write_formula_overlay_enabled
14365                {
14366                    let anchor = self
14367                        .graph
14368                        .get_cell_ref(vertex_id)
14369                        .expect("cell ref for vertex");
14370                    let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14371                    self.mirror_value_to_computed_overlay(
14372                        &sheet_name,
14373                        anchor.coord.row() + 1,
14374                        anchor.coord.col() + 1,
14375                        &err_val,
14376                    );
14377                }
14378                Ok(err_val)
14379            }
14380        }
14381    }
14382
14383    fn evaluate_named_scalar(
14384        &mut self,
14385        vertex_id: VertexId,
14386        sheet_id: SheetId,
14387    ) -> Result<LiteralValue, ExcelError> {
14388        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14389            ExcelError::new(ExcelErrorKind::Name)
14390                .with_message("Named range metadata missing".to_string())
14391        })?;
14392
14393        match &named_range.definition {
14394            NamedDefinition::Cell(cell_ref) => {
14395                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14396                let row = cell_ref.coord.row() + 1;
14397                let col = cell_ref.coord.col() + 1;
14398
14399                if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
14400                    && matches!(
14401                        self.graph.get_vertex_kind(dep_vertex),
14402                        VertexKind::FormulaScalar | VertexKind::FormulaArray
14403                    )
14404                {
14405                    // Graph does not cache cell/formula values; ensure the precedent is evaluated.
14406                    let value = self.evaluate_vertex(dep_vertex)?;
14407                    self.graph.update_vertex_value_ref(vertex_id, &value);
14408                    Ok(value)
14409                } else {
14410                    let value = self
14411                        .get_cell_value(sheet_name, row, col)
14412                        .unwrap_or(LiteralValue::Empty);
14413                    self.graph.update_vertex_value_ref(vertex_id, &value);
14414                    Ok(value)
14415                }
14416            }
14417            NamedDefinition::Literal(v) => {
14418                let out = v.clone();
14419                self.graph.update_vertex_value_ref(vertex_id, &out);
14420                Ok(out)
14421            }
14422            NamedDefinition::Formula { ast, .. } => {
14423                let context_sheet = match named_range.scope {
14424                    NameScope::Sheet(id) => id,
14425                    NameScope::Workbook => sheet_id,
14426                };
14427                let sheet_name = self.graph.sheet_name(context_sheet);
14428                let cell_ref = self
14429                    .graph
14430                    .get_cell_ref(vertex_id)
14431                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14432                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
14433                match interpreter.evaluate_ast(ast) {
14434                    Ok(cv) => {
14435                        let value = cv.into_literal();
14436                        match value {
14437                            LiteralValue::Array(_) => {
14438                                let err = ExcelError::new(ExcelErrorKind::NImpl)
14439                                    .with_message("Array result in scalar named range".to_string());
14440                                let err_val = LiteralValue::Error(err.clone());
14441                                self.graph.update_vertex_value_ref(vertex_id, &err_val);
14442                                Ok(err_val)
14443                            }
14444                            other => {
14445                                self.graph.update_vertex_value_ref(vertex_id, &other);
14446                                Ok(other)
14447                            }
14448                        }
14449                    }
14450                    Err(err) => {
14451                        let err_val = LiteralValue::Error(err.clone());
14452                        self.graph.update_vertex_value_ref(vertex_id, &err_val);
14453                        Ok(err_val)
14454                    }
14455                }
14456            }
14457            NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
14458                .with_message("Range-valued name evaluated as scalar".to_string())),
14459        }
14460    }
14461
14462    fn evaluate_named_array(
14463        &mut self,
14464        vertex_id: VertexId,
14465        sheet_id: SheetId,
14466    ) -> Result<LiteralValue, ExcelError> {
14467        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14468            ExcelError::new(ExcelErrorKind::Name)
14469                .with_message("Named range metadata missing".to_string())
14470        })?;
14471
14472        let out = match &named_range.definition {
14473            NamedDefinition::Range(range_ref) => {
14474                if range_ref.start.sheet_id != range_ref.end.sheet_id {
14475                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14476                        .with_message("Named range cannot span sheets".to_string()));
14477                }
14478
14479                let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
14480                let sr0 = range_ref.start.coord.row();
14481                let sc0 = range_ref.start.coord.col();
14482                let er0 = range_ref.end.coord.row();
14483                let ec0 = range_ref.end.coord.col();
14484                if sr0 > er0 || sc0 > ec0 {
14485                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14486                        .with_message("Invalid named range bounds".to_string()));
14487                }
14488
14489                let h = (er0 - sr0 + 1) as usize;
14490                let w = (ec0 - sc0 + 1) as usize;
14491                let cell_count = (h as u64).saturating_mul(w as u64);
14492                if cell_count > self.config.spill.max_spill_cells as u64 {
14493                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
14494                        "Named range too large to materialize as an array".to_string(),
14495                    ));
14496                }
14497
14498                let mut rows = Vec::with_capacity(h);
14499                for r0 in sr0..=er0 {
14500                    let mut row = Vec::with_capacity(w);
14501                    for c0 in sc0..=ec0 {
14502                        let v = self
14503                            .get_cell_value(sheet_name, r0 + 1, c0 + 1)
14504                            .unwrap_or(LiteralValue::Empty);
14505                        row.push(v);
14506                    }
14507                    rows.push(row);
14508                }
14509                LiteralValue::Array(rows)
14510            }
14511            NamedDefinition::Cell(cell_ref) => {
14512                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14513                let row = cell_ref.coord.row() + 1;
14514                let col = cell_ref.coord.col() + 1;
14515                let v = self
14516                    .get_cell_value(sheet_name, row, col)
14517                    .unwrap_or(LiteralValue::Empty);
14518                LiteralValue::Array(vec![vec![v]])
14519            }
14520            NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
14521            NamedDefinition::Formula { ast, .. } => {
14522                let context_sheet = match named_range.scope {
14523                    NameScope::Sheet(id) => id,
14524                    NameScope::Workbook => sheet_id,
14525                };
14526                let sheet_name = self.graph.sheet_name(context_sheet);
14527                let cell_ref = self
14528                    .graph
14529                    .get_cell_ref(vertex_id)
14530                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14531                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
14532                match interpreter.evaluate_ast(ast) {
14533                    Ok(cv) => {
14534                        let v = cv.into_literal();
14535                        match v {
14536                            LiteralValue::Array(_) => v,
14537                            other => LiteralValue::Array(vec![vec![other]]),
14538                        }
14539                    }
14540                    Err(err) => LiteralValue::Error(err),
14541                }
14542            }
14543        };
14544
14545        self.graph.update_vertex_value_ref(vertex_id, &out);
14546        Ok(out)
14547    }
14548
14549    fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
14550        crate::engine::ResourceLedgerError::Exhausted(
14551            formualizer_common::ResourceExhaustionDetail {
14552                reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
14553                limit: limit as u64,
14554                observed: limit.saturating_add(1) as u64,
14555                request_id: self
14556                    .active_evaluation_resource_request
14557                    .as_ref()
14558                    .map(|stats| stats.request_id),
14559            },
14560        )
14561        .into_excel_error()
14562        .with_message(format!("{context} did not converge after {limit} replans"))
14563    }
14564
14565    fn transient_target_preparation_stale(error: &ExcelError) -> bool {
14566        matches!(
14567            &error.extra,
14568            formualizer_common::ExcelErrorExtra::PreparationStale {
14569                reason: formualizer_common::PreparationStaleReason::Semantic
14570                    | formualizer_common::PreparationStaleReason::Provider
14571            }
14572        )
14573    }
14574
14575    fn prepare_graph_for_routed_evaluation(
14576        &mut self,
14577        targets: &[crate::engine::EvaluationTarget],
14578        options: &crate::engine::TargetEvalOptions<'_>,
14579    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
14580        const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
14581        let mut retries = 0usize;
14582        loop {
14583            match self.prepare_graph_for_targets_unobserved(targets, options) {
14584                Err(error)
14585                    if Self::transient_target_preparation_stale(&error)
14586                        && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
14587                {
14588                    retries = retries.saturating_add(1);
14589                }
14590                result => return result,
14591            }
14592        }
14593    }
14594
14595    fn evaluate_mixed_targets(
14596        &mut self,
14597        targets: &[crate::engine::EvaluationTarget],
14598        delta: Option<&mut DeltaCollector>,
14599    ) -> Result<EvalResult, ExcelError> {
14600        let _source_cache = self.source_cache_session();
14601        let cancel = self.active_cancel_flag.clone();
14602        let options = crate::engine::TargetEvalOptions {
14603            request_id: self
14604                .active_evaluation_resource_request
14605                .as_ref()
14606                .map(|stats| stats.request_id),
14607            cancel,
14608            deadline: None,
14609            budgets: None,
14610            opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
14611        };
14612        self.prepare_and_execute_target_recipe(targets, &options, delta)
14613    }
14614
14615    fn prepare_and_execute_target_recipe(
14616        &mut self,
14617        targets: &[crate::engine::EvaluationTarget],
14618        options: &crate::engine::TargetEvalOptions<'_>,
14619        delta: Option<&mut DeltaCollector>,
14620    ) -> Result<EvalResult, ExcelError> {
14621        let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
14622        self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
14623    }
14624
14625    fn execute_prepared_target_recipe(
14626        &mut self,
14627        targets: &[crate::engine::EvaluationTarget],
14628        scope: &crate::engine::PrepareScope,
14629        delta: Option<&mut DeltaCollector>,
14630    ) -> Result<EvalResult, ExcelError> {
14631        self.require_unified_authority()?;
14632        if matches!(scope, crate::engine::PrepareScope::Workbook)
14633            && let Some(stats) = self.active_evaluation_resource_request.as_mut()
14634        {
14635            stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
14636        }
14637        let mut roots = self.resolve_target_producers(targets)?;
14638        if let crate::engine::PrepareScope::Sheets(sheets) = scope {
14639            let request_id = self
14640                .active_evaluation_resource_request
14641                .as_ref()
14642                .map(|request| request.request_id);
14643            let root_count = roots.len();
14644            let mut widened_roots =
14645                OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
14646                    .map_err(|_| target_root_allocation_error(root_count, request_id))?;
14647            let sheet_ids = sheets
14648                .iter()
14649                .filter_map(|sheet| self.graph.sheet_id(sheet))
14650                .collect::<FxHashSet<_>>();
14651            for vertex in self.graph.formula_vertices() {
14652                if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
14653                    widened_roots
14654                        .push(crate::engine::target_preparation::TargetProducer::Legacy(
14655                            vertex,
14656                        ))
14657                        .map_err(|_| {
14658                            target_root_allocation_error(widened_roots.len() + 1, request_id)
14659                        })?;
14660                }
14661            }
14662            roots = widened_roots.into_vec();
14663        }
14664        self.begin_evaluation_request();
14665        #[cfg(any(test, feature = "legacy_oracle"))]
14666        self.graph.flush_pending_edge_deltas();
14667        let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
14668        if workbook_scope {
14669            if let Some(delta) = delta {
14670                self.evaluate_all_with_delta_collector(delta)
14671            } else {
14672                self.evaluate_all_legacy_impl()
14673            }
14674        } else {
14675            self.evaluate_legacy_target_roots(&roots, delta)
14676        }
14677    }
14678
14679    fn legacy_coordinate_targets(
14680        &mut self,
14681        targets: &[(&str, u32, u32)],
14682    ) -> Vec<crate::engine::EvaluationTarget> {
14683        targets
14684            .iter()
14685            .map(|(sheet, row, col)| {
14686                // Compatibility APIs historically interned an unknown target sheet
14687                // and returned an empty value rather than rejecting the target.
14688                self.graph.sheet_id_mut(sheet);
14689                crate::engine::EvaluationTarget::Cell {
14690                    sheet: (*sheet).to_string(),
14691                    row: *row,
14692                    col: *col,
14693                }
14694            })
14695            .collect()
14696    }
14697
14698    /// Evaluate the necessary mixed producer closure for typed cell, range, name, and table targets.
14699    pub fn evaluate_targets(
14700        &mut self,
14701        targets: &[crate::engine::EvaluationTarget],
14702    ) -> Result<EvalResult, ExcelError> {
14703        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14704            engine.observe_function_semantic_epoch()?;
14705            engine.validate_deterministic_mode()?;
14706            engine.evaluate_mixed_targets(targets, None)
14707        })
14708    }
14709
14710    /// Evaluate typed targets with explicit preparation policy and request controls.
14711    pub fn evaluate_targets_with_options(
14712        &mut self,
14713        targets: &[crate::engine::EvaluationTarget],
14714        options: crate::engine::TargetEvalOptions<'_>,
14715    ) -> Result<EvalResult, ExcelError> {
14716        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14717            engine.active_cancel_flag = options.cancel.clone();
14718            engine.active_evaluation_deadline = options.deadline;
14719            let result = (|| {
14720                engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
14721                engine.observe_function_semantic_epoch()?;
14722                engine.validate_deterministic_mode()?;
14723                let _source_cache = engine.source_cache_session();
14724                engine.prepare_and_execute_target_recipe(targets, &options, None)
14725            })();
14726            engine.active_cancel_flag = None;
14727            engine.active_evaluation_deadline = None;
14728            result
14729        })
14730    }
14731
14732    /// Evaluate typed targets and return the versioned run/region delta for the request.
14733    pub fn evaluate_targets_with_delta(
14734        &mut self,
14735        targets: &[crate::engine::EvaluationTarget],
14736    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
14737        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
14738            engine.observe_function_semantic_epoch()?;
14739            engine.validate_deterministic_mode()?;
14740            let mut collector = DeltaCollector::new(DeltaMode::Cells);
14741            let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
14742            Ok((result, collector.finish_target()))
14743        })
14744    }
14745
14746    /// Evaluate only the necessary precedents for specific target cells (demand-driven)
14747    pub fn evaluate_until(
14748        &mut self,
14749        targets: &[(&str, u32, u32)],
14750    ) -> Result<EvalResult, ExcelError> {
14751        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14752            engine.evaluate_until_unobserved(targets)
14753        })
14754    }
14755
14756    fn evaluate_until_unobserved(
14757        &mut self,
14758        targets: &[(&str, u32, u32)],
14759    ) -> Result<EvalResult, ExcelError> {
14760        self.observe_function_semantic_epoch()?;
14761        let targets = self.legacy_coordinate_targets(targets);
14762        self.evaluate_mixed_targets(&targets, None)
14763    }
14764
14765    fn evaluate_until_with_delta_collector(
14766        &mut self,
14767        targets: &[(&str, u32, u32)],
14768        delta: &mut DeltaCollector,
14769    ) -> Result<EvalResult, ExcelError> {
14770        let targets = self.legacy_coordinate_targets(targets);
14771        self.evaluate_mixed_targets(&targets, Some(delta))
14772    }
14773
14774    fn evaluate_legacy_target_roots(
14775        &mut self,
14776        roots: &[crate::engine::target_preparation::TargetProducer],
14777        mut delta: Option<&mut DeltaCollector>,
14778    ) -> Result<EvalResult, ExcelError> {
14779        use crate::engine::target_preparation::TargetProducer;
14780        #[cfg(any(test, feature = "benchmark_internal"))]
14781        {
14782            self.recalc_reuse_probe
14783                .get_mut()
14784                .unwrap()
14785                .legacy_target_requests += 1;
14786        }
14787        let start = crate::instant::FzInstant::now();
14788        let root_vertices = roots
14789            .iter()
14790            .filter_map(|root| match root {
14791                TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
14792                TargetProducer::ValueOnly(_) => None,
14793            })
14794            .collect::<Vec<_>>();
14795        let mut computed_vertices = 0usize;
14796        let mut cycle_errors = 0usize;
14797        let mut replans = 0usize;
14798        const MAX_REPLAN: usize = 5;
14799        self.graph.authority_sync();
14800        loop {
14801            let (precedents_to_eval, old_vdeps) = self.demand_subgraph(&root_vertices)?;
14802            if precedents_to_eval.is_empty() {
14803                break;
14804            }
14805            #[cfg(any(test, feature = "benchmark_internal"))]
14806            {
14807                self.recalc_reuse_probe
14808                    .get_mut()
14809                    .unwrap()
14810                    .target_schedule_builds += 1;
14811            }
14812            let schedule = {
14813                self.graph.authority_sync();
14814                let mut ledger = self.active_resource_ledger.take();
14815                let result = self.create_authority_schedule(
14816                    &precedents_to_eval,
14817                    &old_vdeps,
14818                    ledger.as_mut(),
14819                );
14820                self.active_resource_ledger = ledger;
14821                result?
14822            };
14823            self.begin_pass(&schedule);
14824            for (unit_index, &unit) in schedule.units.iter().enumerate() {
14825                self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
14826                match unit {
14827                    ScheduleUnit::Cycle(index) => {
14828                        if self.handle_cycle_unit(
14829                            schedule.unit_cycle(index),
14830                            delta.as_deref_mut(),
14831                            None,
14832                            None,
14833                        )? > 0
14834                        {
14835                            cycle_errors = cycle_errors.saturating_add(1);
14836                        }
14837                    }
14838                    ScheduleUnit::Layer(index) => {
14839                        let layer = schedule.unit_layer(index);
14840                        let evaluated = if let Some(delta) = delta.as_deref_mut() {
14841                            if self.thread_pool.is_some() && layer.vertices.len() > 1 {
14842                                self.evaluate_layer_parallel_with_delta(layer, delta)?
14843                            } else {
14844                                self.evaluate_layer_sequential_with_delta(layer, delta)?
14845                            }
14846                        } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
14847                            self.evaluate_layer_parallel(layer)?
14848                        } else {
14849                            self.evaluate_layer_sequential(layer)?
14850                        };
14851                        computed_vertices = computed_vertices.saturating_add(evaluated);
14852                    }
14853                }
14854                if self.stop_after_unit(&schedule, unit_index) {
14855                    break;
14856                }
14857            }
14858            let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
14859            self.resource_checkpoint(0)?;
14860            if !self.finish_target_pass_dirty(&precedents_to_eval, &changed) {
14861                break;
14862            }
14863            if replans >= MAX_REPLAN {
14864                return Err(self.replan_exhausted_error(
14865                    MAX_REPLAN,
14866                    "targeted legacy dynamic dependency evaluation",
14867                ));
14868            }
14869            replans = replans.saturating_add(1);
14870        }
14871        self.redirty_for_next_recalc();
14872        Ok(EvalResult {
14873            computed_vertices,
14874            cycle_errors,
14875            elapsed: start.elapsed(),
14876        })
14877    }
14878
14879    /// Build a revision-bound compatibility plan covering every prepared formula vertex.
14880    pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
14881        if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
14882            return Err(
14883                Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
14884                    "compatibility recalculation plans require all staged formulas to be prepared",
14885                ),
14886            );
14887        }
14888        let key = self.recalc_plan_key();
14889        let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
14890        vertices.sort_unstable();
14891        let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
14892        let schedule = if vertices.is_empty() {
14893            crate::engine::Schedule {
14894                units: Vec::new(),
14895                layers: Vec::new(),
14896                cycles: Vec::new(),
14897            }
14898        } else {
14899            self.create_evaluation_schedule_uncached(&vertices, None)?.0
14900        };
14901        self.validate_recalc_plan_key(&key)?;
14902        Ok(RecalcPlan {
14903            key,
14904            kind: RecalcPlanKind::CompatibilityFull {
14905                schedule,
14906                has_dynamic_refs,
14907            },
14908        })
14909    }
14910
14911    /// Prepare stable typed targets and retain a revision-bound run-local recipe.
14912    pub fn build_recalc_plan_for_targets(
14913        &mut self,
14914        targets: &[crate::engine::EvaluationTarget],
14915    ) -> Result<RecalcPlan, ExcelError> {
14916        self.build_recalc_plan_for_targets_with_options(
14917            targets,
14918            crate::engine::TargetEvalOptions::default(),
14919        )
14920    }
14921
14922    pub fn build_recalc_plan_for_targets_with_options(
14923        &mut self,
14924        targets: &[crate::engine::EvaluationTarget],
14925        options: crate::engine::TargetEvalOptions<'_>,
14926    ) -> Result<RecalcPlan, ExcelError> {
14927        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
14928            engine.observe_function_semantic_epoch()?;
14929            engine.validate_deterministic_mode()?;
14930            let _source_cache = engine.source_cache_session();
14931            let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
14932            #[cfg(any(test, feature = "legacy_oracle"))]
14933            engine.graph.flush_pending_edge_deltas();
14934            let topology = if matches!(
14935                preparation.widened_scope,
14936                crate::engine::PrepareScope::Workbook
14937            ) {
14938                RecalcTopology::Workbook
14939            } else {
14940                RecalcTopology::RunLocalRecipe
14941            };
14942            Ok(RecalcPlan {
14943                key: engine.recalc_plan_key(),
14944                kind: RecalcPlanKind::Target {
14945                    targets: targets.to_vec(),
14946                    scope: preparation.widened_scope,
14947                    topology,
14948                    dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
14949                },
14950            })
14951        })
14952    }
14953
14954    /// Evaluate using a previously constructed compatibility or target plan.
14955    pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
14956        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
14957            engine.evaluate_recalc_plan_unobserved(plan)
14958        })
14959    }
14960
14961    pub fn evaluate_recalc_plan_with_controls(
14962        &mut self,
14963        plan: &RecalcPlan,
14964        cancel: Option<crate::engine::CancelToken>,
14965        deadline: Option<Instant>,
14966    ) -> Result<EvalResult, ExcelError> {
14967        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
14968            engine.active_cancel_flag = cancel.clone();
14969            engine.active_evaluation_deadline = deadline;
14970            let result = engine.evaluate_recalc_plan_unobserved(plan);
14971            engine.active_cancel_flag = None;
14972            engine.active_evaluation_deadline = None;
14973            result
14974        })
14975    }
14976
14977    fn evaluate_recalc_plan_unobserved(
14978        &mut self,
14979        plan: &RecalcPlan,
14980    ) -> Result<EvalResult, ExcelError> {
14981        #[cfg(any(test, feature = "legacy_oracle"))]
14982        self.graph.flush_pending_edge_deltas();
14983        self.validate_recalc_plan_key(&plan.key)?;
14984        self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
14985        self.validate_deterministic_mode()?;
14986
14987        match &plan.kind {
14988            RecalcPlanKind::Target {
14989                targets,
14990                scope,
14991                topology,
14992                dynamic_policy,
14993            } => {
14994                debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
14995                debug_assert_eq!(
14996                    matches!(topology, RecalcTopology::Workbook),
14997                    matches!(scope, crate::engine::PrepareScope::Workbook)
14998                );
14999                let _source_cache = self.source_cache_session();
15000                self.execute_prepared_target_recipe(targets, scope, None)
15001            }
15002            RecalcPlanKind::CompatibilityFull {
15003                schedule,
15004                has_dynamic_refs,
15005            } => {
15006                let _source_cache = self.source_cache_session();
15007                self.begin_evaluation_request();
15008                if *has_dynamic_refs {
15009                    self.virtual_dep_fallback_activations =
15010                        self.virtual_dep_fallback_activations.saturating_add(1);
15011                    return self.evaluate_all_coordinator();
15012                }
15013
15014                let start = crate::instant::FzInstant::now();
15015                let dirty_vertices = self.graph.get_evaluation_vertices();
15016                if dirty_vertices.is_empty() {
15017                    return Ok(EvalResult {
15018                        computed_vertices: 0,
15019                        cycle_errors: 0,
15020                        elapsed: start.elapsed(),
15021                    });
15022                }
15023
15024                let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
15025                let mut computed_vertices = 0;
15026                let mut cycle_errors = 0;
15027                for &unit in &schedule.units {
15028                    self.cancellation_checkpoint(
15029                        "Evaluation cancelled before recalculation plan schedule unit",
15030                    )?;
15031                    match unit {
15032                        ScheduleUnit::Cycle(i) => {
15033                            let stamped = self.handle_cycle_unit(
15034                                schedule.unit_cycle(i),
15035                                None,
15036                                Some(&dirty_set),
15037                                None,
15038                            )?;
15039                            if stamped > 0 {
15040                                cycle_errors += 1;
15041                            }
15042                        }
15043                        ScheduleUnit::Layer(i) => {
15044                            let work: Vec<VertexId> = schedule
15045                                .unit_layer(i)
15046                                .vertices
15047                                .iter()
15048                                .copied()
15049                                .filter(|v| dirty_set.contains(v))
15050                                .collect();
15051                            if work.is_empty() {
15052                                continue;
15053                            }
15054                            let temp_layer = crate::engine::scheduler::Layer::new(work);
15055                            if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
15056                                computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
15057                            } else {
15058                                computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
15059                            }
15060                        }
15061                    }
15062                }
15063
15064                self.resource_checkpoint(0)?;
15065                self.graph.clear_dirty_flags(&dirty_vertices);
15066                self.redirty_for_next_recalc();
15067                Ok(EvalResult {
15068                    computed_vertices,
15069                    cycle_errors,
15070                    elapsed: start.elapsed(),
15071                })
15072            }
15073        }
15074    }
15075}
15076
15077impl<R> Engine<R>
15078where
15079    R: EvaluationContext,
15080{
15081    /// Refuse out-of-scope authority states before evaluation can demote spans
15082    /// or execute a legacy schedule. The public error type is unchanged; NImpl
15083    /// carries the exact internal Unsupported operation for the deferred-scope
15084    /// gate. Admission and allocation failures are not scope exceptions.
15085    fn require_unified_authority(&mut self) -> Result<(), ExcelError> {
15086        self.graph
15087            .authority()
15088            .map(|_| ())
15089            .map_err(Self::authority_excel_error)
15090    }
15091
15092    fn authority_excel_error(error: crate::engine::authority::store::AuthorityError) -> ExcelError {
15093        use crate::engine::authority::store::AuthorityError;
15094        // Some unchanged behavioral tests assert only `error.kind`, hiding
15095        // the operation in their panic. The opt-in gate trace proves which
15096        // typed error was actually returned; it never changes that error.
15097        #[cfg(test)]
15098        if std::env::var_os("FZ_AUTHORITY_DEFERRED_TRACE").is_some() {
15099            eprintln!("M1B_AUTHORITY_ERROR {error:?}");
15100        }
15101        let kind = match error {
15102            AuthorityError::Unsupported { .. } => ExcelErrorKind::NImpl,
15103            _ => ExcelErrorKind::Error,
15104        };
15105        ExcelError::new(kind).with_message(format!("unified_authority: {error:?}"))
15106    }
15107
15108    /// Evaluate all dirty/volatile vertices
15109    /// Evaluate a current result snapshot, optionally with cancellation.
15110    ///
15111    /// This evaluates exactly like [`Self::evaluate_all`] (or its cancellable
15112    /// counterpart), but leaves volatile vertices and their dependents Current
15113    /// at return rather than marking them dirty for the next cycle. It does
15114    /// not suppress iterative-SCC redirty or any other freshness check.
15115    ///
15116    /// Callers retaining the engine must start another evaluation request
15117    /// (`evaluate_all`, `evaluate_until`, `evaluate_cell`, or equivalent) to
15118    /// refresh volatile values. That request restores the deferred volatile
15119    /// redirty before selecting its work. Errors and cancellation never leave
15120    /// the snapshot mode active.
15121    pub fn evaluate_all_for_snapshot(
15122        &mut self,
15123        cancel: Option<crate::engine::CancelToken>,
15124    ) -> Result<EvalResult, ExcelError> {
15125        /// Leaves snapshot mode even when evaluation unwinds. A panic
15126        /// keeps the deferred volatile redirty pending, so the next request
15127        /// restores volatile work without touching the graph mid-unwind.
15128        struct SnapshotScope<'e, R: EvaluationContext>(&'e mut Engine<R>);
15129        impl<R: EvaluationContext> Drop for SnapshotScope<'_, R> {
15130            fn drop(&mut self) {
15131                self.0.snapshot_evaluation_active = false;
15132                if std::thread::panicking() {
15133                    self.0.snapshot_volatile_redirty_pending = true;
15134                }
15135            }
15136        }
15137        self.snapshot_evaluation_active = true;
15138        let result = {
15139            let scope = SnapshotScope(self);
15140            match cancel {
15141                Some(cancel) => scope.0.evaluate_all_cancellable(cancel),
15142                None => scope.0.evaluate_all(),
15143            }
15144        };
15145        if result.is_err() {
15146            self.snapshot_volatile_redirty_pending = false;
15147            self.graph.redirty_volatiles();
15148        }
15149        result
15150    }
15151
15152    /// Evaluate all dirty/volatile vertices.
15153    pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
15154        // `evaluate_all_unobserved` owns the `observe_function_semantic_epoch` guard.
15155        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
15156            engine.evaluate_all_unobserved()
15157        })
15158    }
15159
15160    fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
15161        debug_assert!(
15162            !self.graph.deferred_dirty_active(),
15163            "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
15164             was not balanced by end_deferred_dirty"
15165        );
15166        self.observe_function_semantic_epoch()?;
15167        self.lookup_index_cache.reset_counters();
15168        let _source_cache = self.source_cache_session();
15169        self.validate_deterministic_mode()?;
15170        if self.config.defer_graph_building {
15171            // Build graph for all staged formulas before evaluating
15172            self.build_graph_all()?;
15173        }
15174        self.evaluate_all_coordinator()
15175    }
15176
15177    /// Coordinator for `evaluate_all`: starts the evaluation request and runs
15178    /// the per-cell pass.
15179    fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
15180        self.require_unified_authority()?;
15181        self.begin_evaluation_request();
15182        self.evaluate_all_legacy_impl()
15183    }
15184
15185    /// Walk a schedule's units in condensation order: stamp each cyclic SCC
15186    /// at its position and evaluate each layer (parallel when enabled).
15187    ///
15188    /// Returns `(computed_vertices, cycle_count)` where `cycle_count` is the
15189    /// number of Cycle units walked (the former `schedule.cycles.len()`).
15190    fn legacy_pass_run_units(
15191        &mut self,
15192        schedule: &crate::engine::scheduler::Schedule,
15193    ) -> Result<(usize, usize), ExcelError> {
15194        let mut computed_vertices = 0;
15195        let mut cycle_count = 0;
15196        self.begin_pass(schedule);
15197        for (unit_index, &unit) in schedule.units.iter().enumerate() {
15198            match unit {
15199                ScheduleUnit::Cycle(i) => {
15200                    if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
15201                        cycle_count += 1;
15202                    }
15203                }
15204                ScheduleUnit::Layer(i) => {
15205                    let layer = schedule.unit_layer(i);
15206                    if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15207                        computed_vertices += self.evaluate_layer_parallel(layer)?;
15208                    } else {
15209                        computed_vertices += self.evaluate_layer_sequential(layer)?;
15210                    }
15211                }
15212            }
15213            if self.stop_after_unit(schedule, unit_index) {
15214                break;
15215            }
15216        }
15217        Ok((computed_vertices, cycle_count))
15218    }
15219
15220    /// Per-cell `evaluate_all` body, reached through the coordinator. This is
15221    /// an internal primitive; it must not be invoked directly from public APIs.
15222    ///
15223    /// Does NOT call `begin_evaluation_request` (cycle-telemetry reset +
15224    /// per-recalc clock sample): request begin happens at the public entry
15225    /// points / coordinators, so one request keeps one clock sample.
15226    fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
15227        self.reset_virtual_dep_telemetry_if_disabled();
15228        let _span_eval =
15229            crate::engine::trace::fz_span!(tracing::Level::INFO, "evaluate", "evaluate.legacy");
15230        let start = crate::instant::FzInstant::now();
15231        let mut computed_vertices = 0;
15232        let mut cycle_errors = 0;
15233        let mut replan_iterations = 0;
15234        const MAX_REPLAN: usize = 5;
15235        let mut telemetry = self
15236            .config
15237            .enable_virtual_dep_telemetry
15238            .then(|| self.start_virtual_dep_telemetry());
15239
15240        loop {
15241            let to_evaluate = self.graph.get_evaluation_vertices();
15242            if to_evaluate.is_empty() {
15243                if let Some(t) = telemetry.as_mut()
15244                    && t.bailout_reason.is_none()
15245                {
15246                    t.bailout_reason = Some("no_work");
15247                }
15248                break;
15249            }
15250
15251            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15252            if let Some(t) = telemetry.as_mut() {
15253                Self::accumulate_schedule_meta(t, &meta);
15254            }
15255
15256            let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
15257            computed_vertices += pass_computed;
15258            cycle_errors += pass_cycles;
15259
15260            // Check if dynamic dependencies changed
15261            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15262            if let Some(t) = telemetry.as_mut() {
15263                t.changed_vdeps_total += changed_vertices.len();
15264            }
15265
15266            self.resource_checkpoint(0)?;
15267            if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15268                if let Some(t) = telemetry.as_mut() {
15269                    t.bailout_reason = Some("converged");
15270                }
15271                break;
15272            }
15273            if replan_iterations >= MAX_REPLAN {
15274                if let Some(mut t) = telemetry.take() {
15275                    t.bailout_reason = Some("max_replan");
15276                    t.replan_iterations = replan_iterations;
15277                    self.last_virtual_dep_telemetry = t;
15278                }
15279                return Err(
15280                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15281                );
15282            }
15283
15284            replan_iterations += 1;
15285        }
15286
15287        if let Some(mut t) = telemetry {
15288            t.replan_iterations = replan_iterations;
15289            self.last_virtual_dep_telemetry = t;
15290        }
15291
15292        // Re-dirty volatile vertices for the next evaluation cycle
15293        self.redirty_for_next_recalc();
15294
15295        // Advance recalc epoch after a full evaluation pass finishes
15296        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15297
15298        Ok(EvalResult {
15299            computed_vertices,
15300            cycle_errors,
15301            elapsed: start.elapsed(),
15302        })
15303    }
15304
15305    pub fn evaluate_all_with_target_delta(
15306        &mut self,
15307    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15308        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15309            engine.observe_function_semantic_epoch()?;
15310            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15311            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15312            Ok((result, collector.finish_target()))
15313        })
15314    }
15315
15316    pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
15317        self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
15318    }
15319
15320    pub fn evaluate_all_with_delta_policy(
15321        &mut self,
15322        policy: EvalDeltaCompatibilityPolicy,
15323    ) -> Result<(EvalResult, EvalDelta), ExcelError> {
15324        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15325            engine.observe_function_semantic_epoch()?;
15326            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15327            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15328            Ok((result, collector.finish_with_policy(policy)?))
15329        })
15330    }
15331
15332    fn evaluate_all_with_delta_collector(
15333        &mut self,
15334        delta: &mut DeltaCollector,
15335    ) -> Result<EvalResult, ExcelError> {
15336        let _source_cache = self.source_cache_session();
15337        if self.config.defer_graph_building {
15338            self.build_graph_all()?;
15339        }
15340        self.require_unified_authority()?;
15341        self.begin_evaluation_request();
15342        self.reset_virtual_dep_telemetry_if_disabled();
15343        let _span_eval = crate::engine::trace::fz_span!(
15344            tracing::Level::INFO,
15345            "evaluate",
15346            "evaluate.legacy_delta"
15347        );
15348        let start = crate::instant::FzInstant::now();
15349        let mut computed_vertices = 0;
15350        let mut cycle_errors = 0;
15351
15352        let mut replan_iterations = 0;
15353        const MAX_REPLAN: usize = 5;
15354        let mut telemetry = self
15355            .config
15356            .enable_virtual_dep_telemetry
15357            .then(|| self.start_virtual_dep_telemetry());
15358
15359        loop {
15360            let to_evaluate = self.graph.get_evaluation_vertices();
15361            if to_evaluate.is_empty() {
15362                if let Some(t) = telemetry.as_mut()
15363                    && t.bailout_reason.is_none()
15364                {
15365                    t.bailout_reason = Some("no_work");
15366                }
15367                break;
15368            }
15369
15370            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15371            if let Some(t) = telemetry.as_mut() {
15372                Self::accumulate_schedule_meta(t, &meta);
15373            }
15374
15375            self.begin_pass(&schedule);
15376            for (unit_index, &unit) in schedule.units.iter().enumerate() {
15377                match unit {
15378                    ScheduleUnit::Cycle(i) => {
15379                        if self.handle_cycle_unit(
15380                            schedule.unit_cycle(i),
15381                            Some(delta),
15382                            None,
15383                            None,
15384                        )? > 0
15385                        {
15386                            cycle_errors += 1;
15387                        }
15388                    }
15389                    ScheduleUnit::Layer(i) => {
15390                        let layer = schedule.unit_layer(i);
15391                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15392                            computed_vertices +=
15393                                self.evaluate_layer_parallel_with_delta(layer, delta)?;
15394                        } else {
15395                            computed_vertices +=
15396                                self.evaluate_layer_sequential_with_delta(layer, delta)?;
15397                        }
15398                    }
15399                }
15400                if self.stop_after_unit(&schedule, unit_index) {
15401                    break;
15402                }
15403            }
15404
15405            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15406            if let Some(t) = telemetry.as_mut() {
15407                t.changed_vdeps_total += changed_vertices.len();
15408            }
15409            self.resource_checkpoint(0)?;
15410            if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15411                if let Some(t) = telemetry.as_mut() {
15412                    t.bailout_reason = Some("converged");
15413                }
15414                break;
15415            }
15416            if replan_iterations >= MAX_REPLAN {
15417                if let Some(mut t) = telemetry.take() {
15418                    t.bailout_reason = Some("max_replan");
15419                    t.replan_iterations = replan_iterations;
15420                    self.last_virtual_dep_telemetry = t;
15421                }
15422                return Err(
15423                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15424                );
15425            }
15426            replan_iterations += 1;
15427        }
15428
15429        if let Some(mut t) = telemetry {
15430            t.replan_iterations = replan_iterations;
15431            self.last_virtual_dep_telemetry = t;
15432        }
15433
15434        self.redirty_for_next_recalc();
15435        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15436
15437        Ok(EvalResult {
15438            computed_vertices,
15439            cycle_errors,
15440            elapsed: start.elapsed(),
15441        })
15442    }
15443
15444    /// Convenience: demand-driven evaluation of a single cell by sheet name and row/col.
15445    ///
15446    /// This will evaluate only the minimal set of dirty / volatile precedents required
15447    /// to bring the target cell up-to-date (as if a user asked for that single value),
15448    /// rather than scheduling a full workbook recalc. If the cell is already clean and
15449    /// non-volatile, no vertices will be recomputed.
15450    ///
15451    /// Returns the (possibly newly computed) value stored for the cell afterwards.
15452    /// Empty cells return None. Errors are surfaced via the Result type.
15453    pub fn evaluate_cell(
15454        &mut self,
15455        sheet: &str,
15456        row: u32,
15457        col: u32,
15458    ) -> Result<Option<LiteralValue>, ExcelError> {
15459        self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
15460            engine.evaluate_cell_unobserved(sheet, row, col)
15461        })
15462    }
15463
15464    fn evaluate_cell_unobserved(
15465        &mut self,
15466        sheet: &str,
15467        row: u32,
15468        col: u32,
15469    ) -> Result<Option<LiteralValue>, ExcelError> {
15470        if row == 0 || col == 0 {
15471            return Err(ExcelError::new(ExcelErrorKind::Ref)
15472                .with_message("Row and column must be >= 1".to_string()));
15473        }
15474
15475        let result = self.evaluate_cells(&[(sheet, row, col)])?;
15476
15477        match result.len() {
15478            0 => Ok(None),
15479            1 => {
15480                let v = result.into_iter().next().unwrap();
15481                Ok(v)
15482            }
15483            _ => unreachable!("evaluate_cells returned unexpected length"),
15484        }
15485    }
15486
15487    /// Convenience: demand-driven evaluation of multiple cells; accepts a slice of
15488    /// (sheet, row, col) triples. The union of required dirty / volatile precedents
15489    /// is computed once and evaluated, which is typically faster than calling
15490    /// `evaluate_cell` repeatedly for a related set of targets.
15491    ///
15492    /// Returns the resulting values for each requested target in the same order.
15493    pub fn evaluate_cells(
15494        &mut self,
15495        targets: &[(&str, u32, u32)],
15496    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15497        self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
15498            engine.evaluate_cells_unobserved(targets)
15499        })
15500    }
15501
15502    fn evaluate_cells_unobserved(
15503        &mut self,
15504        targets: &[(&str, u32, u32)],
15505    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15506        self.observe_function_semantic_epoch()?;
15507        debug_assert!(
15508            !self.graph.deferred_dirty_active(),
15509            "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
15510             was not balanced by end_deferred_dirty"
15511        );
15512        self.validate_deterministic_mode()?;
15513        if targets.is_empty() {
15514            return Ok(Vec::new());
15515        }
15516        let typed_targets = self.legacy_coordinate_targets(targets);
15517        self.evaluate_mixed_targets(&typed_targets, None)?;
15518        Ok(targets
15519            .iter()
15520            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15521            .collect())
15522    }
15523
15524    pub fn evaluate_cells_cancellable(
15525        &mut self,
15526        targets: &[(&str, u32, u32)],
15527        cancel: crate::engine::CancelToken,
15528    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15529        self.observe_evaluation_resource_request(
15530            EvaluationRequestKind::CellsCancellable,
15531            |engine| {
15532                engine.observe_function_semantic_epoch()?;
15533                engine.active_cancel_flag = Some(cancel.clone());
15534                let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
15535                engine.active_cancel_flag = None;
15536                res
15537            },
15538        )
15539    }
15540
15541    fn evaluate_cells_cancellable_impl(
15542        &mut self,
15543        targets: &[(&str, u32, u32)],
15544        cancel_flag: &AtomicBool,
15545    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15546        self.validate_deterministic_mode()?;
15547        if targets.is_empty() {
15548            return Ok(Vec::new());
15549        }
15550        if cancel_flag.load(Ordering::Relaxed) {
15551            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
15552                .with_message("Evaluation cancelled before target preparation"));
15553        }
15554        let typed_targets = self.legacy_coordinate_targets(targets);
15555        self.evaluate_mixed_targets(&typed_targets, None)?;
15556        Ok(targets
15557            .iter()
15558            .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
15559            .collect())
15560    }
15561
15562    pub fn evaluate_cells_with_target_delta(
15563        &mut self,
15564        targets: &[(&str, u32, u32)],
15565    ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
15566        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15567            engine.observe_function_semantic_epoch()?;
15568            engine.validate_deterministic_mode()?;
15569            if targets.is_empty() {
15570                return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
15571            }
15572            let mut collector = DeltaCollector::new(DeltaMode::Cells);
15573            engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
15574            let values = targets
15575                .iter()
15576                .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
15577                .collect();
15578            Ok((values, collector.finish_target()))
15579        })
15580    }
15581
15582    pub fn evaluate_cells_with_delta(
15583        &mut self,
15584        targets: &[(&str, u32, u32)],
15585    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15586        self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
15587    }
15588
15589    pub fn evaluate_cells_with_delta_policy(
15590        &mut self,
15591        targets: &[(&str, u32, u32)],
15592        policy: EvalDeltaCompatibilityPolicy,
15593    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15594        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15595            engine.evaluate_cells_with_delta_unobserved(targets, policy)
15596        })
15597    }
15598
15599    fn evaluate_cells_with_delta_unobserved(
15600        &mut self,
15601        targets: &[(&str, u32, u32)],
15602        policy: EvalDeltaCompatibilityPolicy,
15603    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15604        self.observe_function_semantic_epoch()?;
15605        self.validate_deterministic_mode()?;
15606        if targets.is_empty() {
15607            return Ok((Vec::new(), EvalDelta::default()));
15608        }
15609        let mut collector = DeltaCollector::new(DeltaMode::Cells);
15610        self.evaluate_until_with_delta_collector(targets, &mut collector)?;
15611        let values = targets
15612            .iter()
15613            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15614            .collect();
15615        Ok((values, collector.finish_with_policy(policy)?))
15616    }
15617
15618    /// Get the evaluation plan for target cells without actually evaluating them
15619    pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
15620        if targets.is_empty() {
15621            return Ok(EvalPlan {
15622                total_vertices_to_evaluate: 0,
15623                layers: Vec::new(),
15624                cycles_detected: 0,
15625                dirty_count: 0,
15626                volatile_count: 0,
15627                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15628                estimated_parallel_layers: 0,
15629                target_cells: Vec::new(),
15630            });
15631        }
15632        if self.config.defer_graph_building && self.has_staged_formulas() {
15633            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
15634                "Evaluation plan requested with deferred graph; build first or call evaluate_*",
15635            ));
15636        }
15637
15638        // Convert targets to A1 notation for consistency
15639        let addresses: Vec<String> = targets
15640            .iter()
15641            .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
15642            .collect();
15643
15644        // Parse target cell addresses
15645        let mut target_addrs = Vec::new();
15646        for (sheet, row, col) in targets {
15647            if let Some(sheet_id) = self.graph.sheet_id(sheet) {
15648                let coord = Coord::from_excel(*row, *col, true, true);
15649                target_addrs.push(CellRef::new(sheet_id, coord));
15650            }
15651        }
15652
15653        // Find vertex IDs for targets
15654        let mut target_vertex_ids = Vec::new();
15655        for addr in &target_addrs {
15656            if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
15657                target_vertex_ids.push(vertex_id);
15658            }
15659        }
15660
15661        if target_vertex_ids.is_empty() {
15662            return Ok(EvalPlan {
15663                total_vertices_to_evaluate: 0,
15664                layers: Vec::new(),
15665                cycles_detected: 0,
15666                dirty_count: 0,
15667                volatile_count: 0,
15668                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15669                estimated_parallel_layers: 0,
15670                target_cells: addresses,
15671            });
15672        }
15673
15674        // Build demand subgraph with virtual edges (same as evaluate_until)
15675        let (precedents_to_eval, vdeps) = self.demand_subgraph(&target_vertex_ids)?;
15676
15677        if precedents_to_eval.is_empty() {
15678            return Ok(EvalPlan {
15679                total_vertices_to_evaluate: 0,
15680                layers: Vec::new(),
15681                cycles_detected: 0,
15682                dirty_count: 0,
15683                volatile_count: 0,
15684                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15685                estimated_parallel_layers: 0,
15686                target_cells: addresses,
15687            });
15688        }
15689
15690        // Count dirty and volatile vertices
15691        let mut dirty_count = 0;
15692        let mut volatile_count = 0;
15693        for &vertex_id in &precedents_to_eval {
15694            if self.graph.is_dirty(vertex_id) {
15695                dirty_count += 1;
15696            }
15697            if self.graph.is_volatile(vertex_id) {
15698                volatile_count += 1;
15699            }
15700        }
15701
15702        // Create schedule for the minimal subgraph honoring virtual edges
15703        let schedule = self.create_authority_schedule(&precedents_to_eval, &vdeps, None)?;
15704
15705        // Build layer information
15706        let mut layers = Vec::new();
15707        let mut estimated_parallel_layers = 0;
15708        let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
15709
15710        for layer in &schedule.layers {
15711            let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
15712            if parallel_eligible {
15713                estimated_parallel_layers += 1;
15714            }
15715
15716            // Get sample cell addresses (up to 5)
15717            let sample_cells: Vec<String> = layer
15718                .vertices
15719                .iter()
15720                .take(5)
15721                .filter_map(|&vertex_id| {
15722                    self.graph
15723                        .get_cell_ref_for_vertex(vertex_id)
15724                        .map(|cell_ref| {
15725                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
15726                            format!(
15727                                "{}!{}{}",
15728                                sheet_name,
15729                                Self::col_to_letters(cell_ref.coord.col().saturating_add(1)),
15730                                cell_ref.coord.row() + 1
15731                            )
15732                        })
15733                })
15734                .collect();
15735
15736            layers.push(LayerInfo {
15737                vertex_count: layer.vertices.len(),
15738                parallel_eligible,
15739                sample_cells,
15740            });
15741        }
15742
15743        Ok(EvalPlan {
15744            total_vertices_to_evaluate: precedents_to_eval.len(),
15745            layers,
15746            cycles_detected: schedule.cycles.len(),
15747            dirty_count,
15748            volatile_count,
15749            parallel_enabled,
15750            estimated_parallel_layers,
15751            target_cells: addresses,
15752        })
15753    }
15754    /// Helper to create a schedule, integrating virtual dependencies automatically.
15755    fn create_evaluation_schedule(
15756        &mut self,
15757        to_evaluate: &[VertexId],
15758    ) -> Result<EvaluationScheduleBuildOutput, ExcelError> {
15759        #[cfg(any(test, feature = "benchmark_internal"))]
15760        {
15761            self.recalc_reuse_probe.get_mut().unwrap().schedule_requests += 1;
15762        }
15763        // Fold pending edge deltas once per schedule build so traversal uses
15764        // the zero-allocation CSR slices (#125).
15765        #[cfg(any(test, feature = "legacy_oracle"))]
15766        self.graph.flush_pending_edge_deltas();
15767        // The cache key includes the authority revision: sync first.
15768        self.graph.authority_sync();
15769        if self.can_use_static_schedule_cache(to_evaluate) {
15770            // A recent schedule for the same request becomes the current one.
15771            let revision = self.schedule_cache_authority_revision();
15772            let current = |e: &CachedScheduleEntry| {
15773                e.topology_epoch == self.topology_epoch && e.authority_revision == revision
15774            };
15775            if !self
15776                .cached_static_schedule
15777                .as_ref()
15778                .is_some_and(|c| current(c) && c.candidate_vertices.equals(to_evaluate))
15779                && let Some(i) = self.recent_schedules.iter().position(|e| {
15780                    current(e)
15781                        && e.candidate_vertices.len() == to_evaluate.len()
15782                        && e.candidate_vertices.equals(to_evaluate)
15783                })
15784            {
15785                let hit = self.recent_schedules.remove(i);
15786                if let Some(previous) = self.cached_static_schedule.replace(hit) {
15787                    self.retain_recent_schedule(previous);
15788                }
15789            }
15790            if let Some(cached) = self.cached_static_schedule.as_ref()
15791                && cached.topology_epoch == self.topology_epoch
15792                && cached.authority_revision == self.schedule_cache_authority_revision()
15793                && cached.candidate_vertices.equals(to_evaluate)
15794            {
15795                let meta = ScheduleBuildMeta {
15796                    candidate_vertices: to_evaluate.len(),
15797                    vdeps_vertices: 0,
15798                    vdeps_edges: 0,
15799                    builder_elapsed_ms: 0,
15800                    used_virtual_schedule: false,
15801                    schedule_cache_hit: true,
15802                    schedule_cache_eligible: true,
15803                };
15804                #[cfg(any(test, feature = "benchmark_internal"))]
15805                {
15806                    let mut probe = self.recalc_reuse_probe.lock().unwrap();
15807                    probe.schedule_cache_hits += 1;
15808                    probe.schedule_shared_handles += 1;
15809                }
15810                return Ok((
15811                    EvaluationSchedule::Shared(Arc::clone(&cached.schedule)),
15812                    FxHashMap::default(),
15813                    meta,
15814                ));
15815            }
15816
15817            let (schedule, vdeps, mut meta) = match self.schedule_from_base(to_evaluate)? {
15818                Some(schedule) => (
15819                    schedule,
15820                    FxHashMap::default(),
15821                    ScheduleBuildMeta {
15822                        candidate_vertices: to_evaluate.len(),
15823                        vdeps_vertices: 0,
15824                        vdeps_edges: 0,
15825                        builder_elapsed_ms: 0,
15826                        used_virtual_schedule: false,
15827                        schedule_cache_hit: false,
15828                        schedule_cache_eligible: true,
15829                    },
15830                ),
15831                None => self.create_evaluation_schedule_active(to_evaluate)?,
15832            };
15833            meta.schedule_cache_hit = false;
15834            meta.schedule_cache_eligible = true;
15835            #[cfg(any(test, feature = "benchmark_internal"))]
15836            {
15837                self.recalc_reuse_probe
15838                    .get_mut()
15839                    .unwrap()
15840                    .schedule_cache_misses += 1;
15841            }
15842            let schedule = if vdeps.is_empty() {
15843                // Clone previously discarded builder spare capacity. Keep that compact
15844                // retained payload while sharing it with the current request.
15845                let mut schedule = schedule;
15846                schedule.units.shrink_to_fit();
15847                for layer in &mut schedule.layers {
15848                    layer.vertices.shrink_to_fit();
15849                }
15850                schedule.layers.shrink_to_fit();
15851                for cycle in &mut schedule.cycles {
15852                    cycle.shrink_to_fit();
15853                }
15854                schedule.cycles.shrink_to_fit();
15855                let schedule = Arc::new(schedule);
15856                #[cfg(any(test, feature = "benchmark_internal"))]
15857                {
15858                    self.recalc_reuse_probe
15859                        .get_mut()
15860                        .unwrap()
15861                        .schedule_shared_handles += 1;
15862                }
15863                let entry = CachedScheduleEntry {
15864                    topology_epoch: self.topology_epoch,
15865                    authority_revision: self.schedule_cache_authority_revision(),
15866                    candidate_vertices: VertexIdRuns::from_slice(to_evaluate),
15867                    schedule: Arc::clone(&schedule),
15868                };
15869                if let Some(previous) = self.cached_static_schedule.replace(entry) {
15870                    self.retain_recent_schedule(previous);
15871                }
15872                EvaluationSchedule::Shared(schedule)
15873            } else {
15874                EvaluationSchedule::Owned(schedule)
15875            };
15876            return Ok((schedule, vdeps, meta));
15877        }
15878
15879        let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_active(to_evaluate)?;
15880        meta.schedule_cache_hit = false;
15881        meta.schedule_cache_eligible = false;
15882        #[cfg(any(test, feature = "benchmark_internal"))]
15883        {
15884            self.recalc_reuse_probe
15885                .get_mut()
15886                .unwrap()
15887                .schedule_cache_ineligible += 1;
15888        }
15889        Ok((EvaluationSchedule::Owned(schedule), vdeps, meta))
15890    }
15891
15892    /// Plan reuse: the base schedule restricted to `to_evaluate` when it
15893    /// is current, covers the request, has at least
15894    /// `BASE_SCHEDULE_MIN_REQUEST` candidates, and is at most
15895    /// `BASE_SCHEDULE_RATIO` times its size (restricting walks the whole
15896    /// base; planning costs far more per candidate).
15897    fn schedule_from_base(
15898        &mut self,
15899        to_evaluate: &[VertexId],
15900    ) -> Result<Option<crate::engine::scheduler::Schedule>, ExcelError> {
15901        let revision = self.schedule_cache_authority_revision();
15902        let current = |e: &&CachedScheduleEntry| {
15903            e.topology_epoch == self.topology_epoch && e.authority_revision == revision
15904        };
15905        // The base, or the current schedule when larger (the first
15906        // evaluation's, before a later request replaces it).
15907        let Some(base) = [
15908            self.base_schedule.as_ref(),
15909            self.cached_static_schedule.as_ref(),
15910        ]
15911        .into_iter()
15912        .flatten()
15913        .filter(current)
15914        .max_by_key(|e| e.candidate_vertices.len()) else {
15915            return Ok(None);
15916        };
15917        let base_len = base.candidate_vertices.len();
15918        if base_len <= BASE_SCHEDULE_MIN_VERTICES
15919            || to_evaluate.len() < BASE_SCHEDULE_MIN_REQUEST
15920            || to_evaluate.len() > base_len
15921            || base_len > to_evaluate.len().saturating_mul(BASE_SCHEDULE_RATIO)
15922        {
15923            return Ok(None);
15924        }
15925        let mut keep = crate::engine::idset::DenseIdSet::default();
15926        keep.extend(to_evaluate.iter().copied());
15927        let Some((schedule, kept)) = base.schedule.restrict(&keep) else {
15928            return Ok(None);
15929        };
15930        // Every requested vertex must be in the base.
15931        if kept != keep.len() {
15932            return Ok(None);
15933        }
15934        if let Some(ledger) = self.active_resource_ledger.as_mut() {
15935            let layers: usize = schedule
15936                .layers
15937                .iter()
15938                .map(|l| {
15939                    l.vertices.capacity() * std::mem::size_of::<VertexId>()
15940                        + l.runs.capacity()
15941                            * std::mem::size_of::<crate::engine::scheduler::LayerRun>()
15942                })
15943                .sum();
15944            let bytes = (keep.heap_bytes()
15945                + layers
15946                + schedule.layers.capacity()
15947                    * std::mem::size_of::<crate::engine::scheduler::Layer>()
15948                + schedule.units.capacity()
15949                    * std::mem::size_of::<crate::engine::scheduler::ScheduleUnit>()
15950                + schedule
15951                    .cycles
15952                    .iter()
15953                    .map(|c| c.capacity() * 4)
15954                    .sum::<usize>()) as u64;
15955            ledger
15956                .reserve_schedule_discovery(bytes)
15957                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
15958            ledger
15959                .release_scratch(bytes)
15960                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
15961        }
15962        #[cfg(debug_assertions)]
15963        self.debug_check_restricted_schedule(to_evaluate, &schedule);
15964        #[cfg(any(test, feature = "benchmark_internal"))]
15965        {
15966            self.recalc_reuse_probe
15967                .get_mut()
15968                .unwrap()
15969                .schedule_base_restrictions += 1;
15970        }
15971        Ok(Some(schedule))
15972    }
15973
15974    /// Debug builds: a restricted schedule holds each requested vertex
15975    /// once and orders the request like a freshly planned schedule (every
15976    /// dependency among the request in an earlier unit).
15977    #[cfg(debug_assertions)]
15978    fn debug_check_restricted_schedule(
15979        &self,
15980        to_evaluate: &[VertexId],
15981        schedule: &crate::engine::scheduler::Schedule,
15982    ) {
15983        let mut position: FxHashMap<VertexId, usize> = FxHashMap::default();
15984        for (u, unit) in schedule.units.iter().enumerate() {
15985            let vertices: &[VertexId] = match *unit {
15986                crate::engine::scheduler::ScheduleUnit::Layer(i) => {
15987                    &schedule.unit_layer(i).vertices
15988                }
15989                crate::engine::scheduler::ScheduleUnit::Cycle(i) => schedule.unit_cycle(i),
15990            };
15991            for &v in vertices {
15992                assert!(
15993                    position.insert(v, u).is_none(),
15994                    "vertex twice in a restricted schedule"
15995                );
15996            }
15997        }
15998        assert_eq!(position.len(), to_evaluate.len());
15999        if to_evaluate.len() > 512 {
16000            return;
16001        }
16002        // Every arc among the request (the store's edge images) goes to an
16003        // earlier unit, or within one sequential layer (a chain) or cycle.
16004        let Ok(store) = self.graph.authority_plan_store() else {
16005            return;
16006        };
16007        let cells: Vec<(VertexId, (u16, u32, u32))> = to_evaluate
16008            .iter()
16009            .filter_map(|&v| self.graph.authority_cell_of_vertex(v).map(|c| (v, c)))
16010            .collect();
16011        for &(v, (sheet, row, col)) in &cells {
16012            let Some(owner) = store.owner_at((sheet, row, col)) else {
16013                continue;
16014            };
16015            let Ok(refined) = store.refine_owner_column(owner, col, row, row, None) else {
16016                continue;
16017            };
16018            for piece in &refined.pieces {
16019                for edge in &refined.edges[piece.edge_start..piece.edge_end] {
16020                    let Some(image) = edge.proj.forward(&piece.domain) else {
16021                        continue;
16022                    };
16023                    for &(d, (s2, r2, c2)) in &cells {
16024                        if d == v
16025                            || edge.proj.sheet != s2
16026                            || !(image.r0 <= r2
16027                                && r2 <= image.r1
16028                                && image.c0 <= c2
16029                                && c2 <= image.c1)
16030                        {
16031                            continue;
16032                        }
16033                        let (pd, pv) = (position[&d], position[&v]);
16034                        let same_ok = pd == pv
16035                            && match schedule.units[pv] {
16036                                crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16037                                    schedule.unit_layer(i).sequential
16038                                }
16039                                crate::engine::scheduler::ScheduleUnit::Cycle(_) => true,
16040                            };
16041                        assert!(
16042                            pd < pv || same_ok,
16043                            "restricted schedule orders {v:?} before its precedent {d:?}"
16044                        );
16045                    }
16046                }
16047            }
16048        }
16049    }
16050
16051    /// Compress family formulas once per authority build (see
16052    /// `EvalConfig::formula_compression`), when no staged or deferred
16053    /// formula package can hold arena ids.
16054    fn maybe_compress_formulas(&mut self) {
16055        if !self.config.formula_compression {
16056            return;
16057        }
16058        let builds = self.graph.authority_host().builds;
16059        if self.compressed_at_build == Some(builds) {
16060            return;
16061        }
16062        self.compressed_at_build = Some(builds);
16063        if self.has_staged_formulas() {
16064            // Staged packages hold arena ids: no compaction. Members that
16065            // are already compressed can still leave the per-cell maps.
16066            self.graph.virtualize_family_members();
16067            return;
16068        }
16069        let pool = self.thread_pool.clone();
16070        let (_, garbage) = self.graph.compress_family_formulas(pool.as_deref());
16071        self.graph.virtualize_family_members();
16072        // Freeing the dropped members' reference texts (one allocation each)
16073        // is most of compaction; with a pool it happens off the critical
16074        // path.
16075        match pool {
16076            Some(pool) if garbage.len() >= 1024 => pool.spawn(move || drop(garbage)),
16077            _ => drop(garbage),
16078        }
16079    }
16080
16081    fn create_evaluation_schedule_active(
16082        &mut self,
16083        to_evaluate: &[VertexId],
16084    ) -> Result<ScheduleBuildOutput, ExcelError> {
16085        self.graph.authority_sync();
16086        self.maybe_compress_formulas();
16087        let mut ledger = self.active_resource_ledger.take();
16088        let result = self.create_evaluation_schedule_uncached(to_evaluate, ledger.as_mut());
16089        self.active_resource_ledger = ledger;
16090        result
16091    }
16092
16093    fn create_evaluation_schedule_uncached(
16094        &self,
16095        to_evaluate: &[VertexId],
16096        #[allow(unused_variables)] ledger: Option<&mut ResourceLedger>,
16097    ) -> Result<ScheduleBuildOutput, ExcelError> {
16098        #[cfg(any(test, feature = "benchmark_internal"))]
16099        {
16100            self.recalc_reuse_probe.lock().unwrap().schedule_builds += 1;
16101        }
16102        let builder = VirtualDepBuilder::new(self);
16103        #[allow(unused_mut)]
16104        let (mut vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
16105            if self.config.enable_virtual_dep_telemetry {
16106                let build_started = crate::instant::FzInstant::now();
16107                let (vdeps, augmented) = builder.build(to_evaluate);
16108                let builder_elapsed_ms = build_started.elapsed().as_millis();
16109                let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
16110                (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
16111            } else {
16112                let (vdeps, augmented) = builder.build(to_evaluate);
16113                (vdeps, augmented, 0, 0)
16114            };
16115
16116        // Replan hints from stale dynamic reads earlier in this request.
16117        {
16118            self.freshness_merge_hints(to_evaluate, &mut vdeps);
16119            self.freshness_extent_hints(to_evaluate, &mut vdeps);
16120        }
16121        let mut final_evaluate = to_evaluate.to_vec();
16122        if !augmented.is_empty() {
16123            final_evaluate.extend(augmented);
16124            final_evaluate.sort_unstable();
16125            final_evaluate.dedup();
16126        }
16127
16128        let use_virtual = !vdeps.is_empty();
16129
16130        let schedule = self.create_authority_schedule(&final_evaluate, &vdeps, ledger)?;
16131
16132        let meta = ScheduleBuildMeta {
16133            candidate_vertices: to_evaluate.len(),
16134            vdeps_vertices: vdeps.len(),
16135            vdeps_edges,
16136            builder_elapsed_ms,
16137            used_virtual_schedule: use_virtual,
16138            schedule_cache_hit: false,
16139            schedule_cache_eligible: false,
16140        };
16141
16142        Ok((schedule, vdeps, meta))
16143    }
16144
16145    fn create_authority_schedule(
16146        &self,
16147        candidates: &[VertexId],
16148        vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16149        mut ledger: Option<&mut ResourceLedger>,
16150    ) -> Result<crate::engine::scheduler::Schedule, ExcelError> {
16151        use crate::engine::authority::{
16152            geom::{Cover, Rect},
16153            plan_schedule, planner,
16154            proj::{AxisMap, RefProj},
16155            store::{EdgeKey, Tag},
16156        };
16157        let failure = |message: String| {
16158            ExcelError::new(ExcelErrorKind::Error)
16159                .with_message(format!("unified_authority planner: {message}"))
16160        };
16161        self.cancellation_checkpoint("Evaluation cancelled before authority planning")?;
16162        let store = self
16163            .graph
16164            .authority_plan_store()
16165            .map_err(Self::authority_excel_error)?;
16166        // One formula cell without hints (a tiny edit): its plan is the cell
16167        // alone unless it reads itself (`planner::plan_single`).
16168        if let [only] = candidates
16169            && vdeps.is_empty()
16170            && self.graph.authority_host().observed(*only).is_none()
16171            && let Some(cell) = self.graph.authority_cell_of_vertex(*only)
16172            && cell.0 != crate::engine::authority::geom::SYMBOL_SHEET
16173            && let Some(single) = planner::plan_single(store, cell)
16174        {
16175            #[cfg(debug_assertions)]
16176            {
16177                let mut cover = Cover::new();
16178                cover.insert_rect(cell.0, &Rect::new(cell.1, cell.2, cell.1, cell.2));
16179                let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16180                    .expect("general plan of one cell");
16181                assert_eq!(
16182                    general.cells.as_slice(),
16183                    &[single],
16184                    "single-cell plan differs from the planner at {cell:?}"
16185                );
16186            }
16187            let adapted = plan_schedule::schedule(
16188                &[single],
16189                0,
16190                None,
16191                |cell| {
16192                    self.graph
16193                        .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16194                        .ok_or_else(|| failure("missing executor identity".to_owned()))
16195                },
16196                |_work| Ok(()),
16197            )
16198            .map_err(|error| match error {
16199                plan_schedule::ScheduleError::Runtime(error) => error,
16200                other => failure(format!("{other:?}")),
16201            })?;
16202            if let Some(ledger) = ledger {
16203                ledger
16204                    .reserve_schedule_discovery(adapted.peak_heap_bytes)
16205                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16206                ledger
16207                    .release_scratch(adapted.peak_heap_bytes)
16208                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16209            }
16210            return Ok(adapted.schedule);
16211        }
16212        // A small request of grid formula cells without hints: per-cell
16213        // arcs and longest-path levels (`planner::plan_small`).
16214        if (2..=planner::SMALL_PLAN_MAX).contains(&candidates.len())
16215            && vdeps.is_empty()
16216            && candidates
16217                .iter()
16218                .all(|&v| self.graph.authority_host().observed(v).is_none())
16219        {
16220            let cells: Option<Vec<(u16, u32, u32)>> = candidates
16221                .iter()
16222                .map(|&v| self.graph.authority_cell_of_vertex(v))
16223                .collect();
16224            if let Some(small) = cells.as_deref().and_then(|c| planner::plan_small(store, c)) {
16225                #[cfg(debug_assertions)]
16226                {
16227                    let mut cover = Cover::new();
16228                    for c in cells.as_deref().unwrap_or_default() {
16229                        cover.insert_rect(c.0, &Rect::new(c.1, c.2, c.1, c.2));
16230                    }
16231                    let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16232                        .expect("general plan of a small request");
16233                    let key = |c: &planner::OrderedCell| (c.sheet, c.row, c.col, c.id, c.owner);
16234                    let mut a: Vec<_> = general.cells.iter().map(key).collect();
16235                    let mut b: Vec<_> = small.iter().map(key).collect();
16236                    a.sort_unstable();
16237                    b.sort_unstable();
16238                    assert_eq!(a, b, "small plan cells differ from the planner");
16239                    assert!(
16240                        general.cells.iter().all(|c| c.cycle.is_none()),
16241                        "small plan of a cyclic request"
16242                    );
16243                }
16244                let adapted = plan_schedule::schedule(
16245                    &small,
16246                    0,
16247                    None,
16248                    |cell| {
16249                        self.graph
16250                            .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16251                            .ok_or_else(|| failure("missing executor identity".to_owned()))
16252                    },
16253                    |_work| Ok(()),
16254                )
16255                .map_err(|error| match error {
16256                    plan_schedule::ScheduleError::Runtime(error) => error,
16257                    other => failure(format!("{other:?}")),
16258                })?;
16259                if let Some(ledger) = ledger {
16260                    ledger
16261                        .reserve_schedule_discovery(adapted.peak_heap_bytes)
16262                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16263                    ledger
16264                        .release_scratch(adapted.peak_heap_bytes)
16265                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16266                }
16267                return Ok(adapted.schedule);
16268            }
16269        }
16270        // Names are symbol-plane nodes (design §4.1): a name vertex plans as
16271        // the unit at its node, between its precedents and its readers.
16272        // Candidates become cells, sorted by (sheet, column, row) and
16273        // coalesced into row intervals: one cover insert per interval.
16274        let mut cover = Cover::new();
16275        {
16276            let cell_of = |&id: &VertexId| {
16277                self.graph
16278                    .authority_cell_of_vertex(id)
16279                    .map(|(sheet, row, col)| (sheet, col, row))
16280            };
16281            // A full recalc maps and sorts every formula: on the pool when
16282            // there is one (first eval's schedule is serial work otherwise).
16283            let cells: Vec<(u16, u32, u32)> = match self.thread_pool.as_deref() {
16284                Some(pool) if candidates.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16285                    use rayon::prelude::*;
16286                    pool.install(|| {
16287                        let mut cells: Vec<_> = candidates.par_iter().filter_map(cell_of).collect();
16288                        cells.par_sort_unstable();
16289                        cells
16290                    })
16291                }
16292                _ => {
16293                    let mut cells: Vec<_> = candidates.iter().filter_map(cell_of).collect();
16294                    cells.sort_unstable();
16295                    cells
16296                }
16297            };
16298            let mut i = 0;
16299            while i < cells.len() {
16300                let (sheet, col, r0) = cells[i];
16301                let mut r1 = r0;
16302                let mut j = i + 1;
16303                while j < cells.len() && cells[j].0 == sheet && cells[j].1 == col {
16304                    if cells[j].2 > r1 + 1 {
16305                        break;
16306                    }
16307                    r1 = r1.max(cells[j].2);
16308                    j += 1;
16309                }
16310                cover.insert_rect(sheet, &Rect::new(r0, col, r1, col));
16311                i = j;
16312            }
16313        }
16314        let mut hints = Vec::new();
16315        for (&reader, deps) in vdeps {
16316            let Some(reader) = self.graph.authority_cell_of_vertex(reader) else {
16317                continue;
16318            };
16319            for &dependency in deps {
16320                let Some(dep) = self.graph.authority_cell_of_vertex(dependency) else {
16321                    continue;
16322                };
16323                hints.push(planner::PlanHint {
16324                    reader: (reader.0, reader.2, reader.1),
16325                    edge: EdgeKey {
16326                        dep_sheet: reader.0,
16327                        tag: Tag::X,
16328                        lk: u32::MAX,
16329                        proj: RefProj {
16330                            sheet: dep.0,
16331                            rows: AxisMap::fixed(dep.1, dep.1),
16332                            cols: AxisMap::fixed(dep.2, dep.2),
16333                        },
16334                    },
16335                });
16336            }
16337        }
16338        // rdi_dyn: order each dynamic reader after its observed reads.
16339        let host = self.graph.authority_host();
16340        for &id in candidates.iter().filter(|_| host.has_observed()) {
16341            let Some(reads) = host.observed(id) else {
16342                continue;
16343            };
16344            let Some(reader) = self.graph.authority_cell_of_vertex(id) else {
16345                continue;
16346            };
16347            for &(sheet, r0, c0, r1, c1) in reads {
16348                hints.push(planner::PlanHint {
16349                    reader: (reader.0, reader.2, reader.1),
16350                    edge: EdgeKey {
16351                        dep_sheet: reader.0,
16352                        tag: Tag::X,
16353                        lk: u32::MAX,
16354                        proj: RefProj {
16355                            sheet,
16356                            rows: AxisMap::fixed(r0, r1),
16357                            cols: AxisMap::fixed(c0, c1),
16358                        },
16359                    },
16360                });
16361            }
16362        }
16363        hints.sort_unstable_by_key(|hint| hint.reader);
16364        let checkpoint = ledger
16365            .as_ref()
16366            .map_or(0, |ledger| ledger.scratch_checkpoint());
16367        let scratch_limit = ledger
16368            .as_ref()
16369            .and_then(|ledger| ledger.schedule_discovery_limit())
16370            .map(|limit| limit.saturating_sub(checkpoint));
16371        let ordered = planner::plan_with_hints(store, &cover, &hints, scratch_limit, None, None)
16372            .map_err(|error| failure(format!("{error:?}")))?;
16373        // max_work_units is an execution budget. Planning work must be capped
16374        // independently: charging it here changes the observable publication
16375        // boundary (e.g. a spill must commit before the next execution fails).
16376        let adapted = plan_schedule::schedule(
16377            &ordered.cells,
16378            ordered.heap_bytes(),
16379            scratch_limit,
16380            |cell| {
16381                self.graph
16382                    .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16383                    .ok_or_else(|| failure("missing executor identity".to_owned()))
16384            },
16385            |_work| {
16386                self.cancellation_checkpoint("Evaluation cancelled during authority planning")?;
16387                if let Some(ledger) = ledger.as_deref_mut() {
16388                    ledger
16389                        .checkpoint_deadline()
16390                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16391                }
16392                Ok(())
16393            },
16394        )
16395        .map_err(|error| match error {
16396            plan_schedule::ScheduleError::Runtime(error) => error,
16397            other => failure(format!("{other:?}")),
16398        })?;
16399        if let Some(ledger) = ledger {
16400            let peak = ordered.peak_heap_bytes.max(adapted.peak_heap_bytes);
16401            ledger
16402                .reserve_schedule_discovery(peak)
16403                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16404            ledger
16405                .release_scratch(peak)
16406                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16407        }
16408        // The planner's order is scratch now; a full recalc's is large.
16409        match self.thread_pool.as_deref() {
16410            Some(pool) if ordered.cells.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16411                pool.spawn(move || drop(ordered))
16412            }
16413            _ => drop(ordered),
16414        }
16415        Ok(adapted.schedule)
16416    }
16417
16418    /// Static-schedule cache eligibility. Legacy excludes range readers:
16419    /// their order comes from per-request range virtual deps. Under the
16420    /// authority a range read is a static edge of the relation, so only
16421    /// dynamic readers (whose hints are per request) are excluded, and the
16422    /// key adds the authority revision (design §8.4).
16423    fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
16424        {
16425            // A dynamic reader is planned from its observed reads, which the
16426            // key covers (rev.dyn); one without them needs a pre-probe, and
16427            // request-scoped replan hints are never cached.
16428            let host = self.graph.authority_host();
16429            !to_evaluate.is_empty()
16430                && !self.freshness_has_hints()
16431                && to_evaluate
16432                    .iter()
16433                    .all(|&v| !self.graph.is_dynamic(v) || host.observed(v).is_some())
16434        }
16435    }
16436
16437    fn schedule_cache_authority_revision(&self) -> (u64, u64) {
16438        {
16439            // rev.topology is `topology_epoch`; a symbol revision rebuilds the
16440            // store, so it is part of `revision`.
16441            let host = self.graph.authority_host();
16442            (host.revision(), host.rev_dyn())
16443        }
16444    }
16445
16446    fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
16447        VirtualDepTelemetry {
16448            fallback_mode_activations: self.virtual_dep_fallback_activations,
16449            ..VirtualDepTelemetry::default()
16450        }
16451    }
16452
16453    fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
16454        telemetry.candidate_vertices_total += meta.candidate_vertices;
16455        telemetry.vdeps_vertices_total += meta.vdeps_vertices;
16456        telemetry.vdeps_edges_total += meta.vdeps_edges;
16457        telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
16458        if meta.schedule_cache_eligible {
16459            if meta.schedule_cache_hit {
16460                telemetry.schedule_cache_hits += 1;
16461                telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
16462            } else {
16463                telemetry.schedule_cache_misses += 1;
16464            }
16465        }
16466        if meta.used_virtual_schedule {
16467            telemetry.schedule_virtual_passes += 1;
16468        } else {
16469            telemetry.schedule_static_passes += 1;
16470        }
16471    }
16472
16473    /// End-of-pass dirty bookkeeping; true when the loop must replan.
16474    /// Legacy: clear the pass, re-dirty readers whose pre-probe changed.
16475    /// Under the authority an armed pass keeps stale and unreached vertices
16476    /// dirty instead (design §8.2, `freshness.rs`).
16477    fn finish_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16478        self.finish_pass_dirty_scoped(to_evaluate, changed, true)
16479    }
16480
16481    /// [`Self::finish_pass_dirty`] for a targeted pass: only its candidates
16482    /// decide whether to replan.
16483    fn finish_target_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16484        self.finish_pass_dirty_scoped(to_evaluate, changed, false)
16485    }
16486
16487    fn finish_pass_dirty_scoped(
16488        &mut self,
16489        to_evaluate: &[VertexId],
16490        changed: &[VertexId],
16491        #[allow(unused_variables)] whole_workbook: bool,
16492    ) -> bool {
16493        self.freshness_finish_pass(to_evaluate, changed, whole_workbook)
16494    }
16495
16496    /// Start a pass over `schedule` (arms the freshness recorder).
16497    fn begin_pass(
16498        &mut self,
16499        #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16500    ) {
16501        self.freshness_begin_pass(schedule);
16502    }
16503
16504    /// FR4 layer barrier after unit `index`: true stops the pass.
16505    fn stop_after_unit(
16506        &mut self,
16507        #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16508        #[allow(unused_variables)] index: usize,
16509    ) -> bool {
16510        self.freshness_stop_after_unit(schedule, index)
16511    }
16512
16513    fn changed_virtual_dep_vertices(
16514        &mut self,
16515        to_evaluate: &[VertexId],
16516        old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16517    ) -> Vec<VertexId> {
16518        #[cfg(test)]
16519        if self.force_virtual_dep_changes_remaining_for_test > 0
16520            && let Some(vertex) = to_evaluate.first().copied()
16521        {
16522            self.force_virtual_dep_changes_remaining_for_test -= 1;
16523            return vec![vertex];
16524        }
16525        // An armed pass detects stale dynamic reads directly; the pre-probe
16526        // comparison is legacy's substitute for that (design §8.2).
16527        if self.freshness_armed() {
16528            return Vec::new();
16529        }
16530        if !to_evaluate
16531            .iter()
16532            .copied()
16533            .any(|v| self.graph.is_dynamic(v))
16534        {
16535            return Vec::new();
16536        }
16537
16538        let builder = VirtualDepBuilder::new(self);
16539        let (new_vdeps, _) = builder.build(to_evaluate);
16540
16541        let mut candidates = FxHashSet::default();
16542        candidates.extend(old_vdeps.keys().copied());
16543        candidates.extend(new_vdeps.keys().copied());
16544
16545        let mut changed = Vec::new();
16546        for v in candidates {
16547            if old_vdeps.get(&v) != new_vdeps.get(&v) {
16548                changed.push(v);
16549            }
16550        }
16551        changed
16552    }
16553
16554    /// Build a demand-driven subgraph for the given targets, including ephemeral edges for
16555    /// compressed ranges, and returning the set of dirty/volatile precedents and virtual deps.
16556    /// Demand candidates of `targets` and their dynamic plan hints: under
16557    /// `unified_authority` from the authority's relation (design §8.3),
16558    /// otherwise from the legacy graph.
16559    #[allow(clippy::type_complexity)]
16560    fn demand_subgraph(
16561        &self,
16562        targets: &[VertexId],
16563    ) -> Result<
16564        (
16565            Vec<VertexId>,
16566            rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16567        ),
16568        ExcelError,
16569    > {
16570        self.authority_demand_subgraph(targets)
16571    }
16572
16573    /// Design §8.3: traverse precedents from the targets over the
16574    /// authority's static relation (symbol nodes are ordinary pieces) plus
16575    /// the dynamic readers' virtual dependencies, and collect what legacy's
16576    /// demand walk collects: dirty or volatile formula cells and every name
16577    /// passed through. No legacy dependency structure is read.
16578    #[allow(clippy::type_complexity)]
16579    fn authority_demand_subgraph(
16580        &self,
16581        targets: &[VertexId],
16582    ) -> Result<
16583        (
16584            Vec<VertexId>,
16585            rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16586        ),
16587        ExcelError,
16588    > {
16589        use crate::engine::authority::geom::{Cell, Rect, SYMBOL_SHEET};
16590        use crate::engine::authority::store::TagFilter;
16591        use rustc_hash::{FxHashMap, FxHashSet};
16592        let store = self
16593            .graph
16594            .authority_plan_store()
16595            .map_err(Self::authority_excel_error)?;
16596        let ids = store.ids();
16597        let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
16598        let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default();
16599        let mut visited: FxHashSet<Cell> = FxHashSet::default();
16600        // (cell, authority id or NO_VID): the id lets the side array
16601        // translate the cell without a hash lookup.
16602        let mut stack: Vec<(Cell, u32)> = Vec::new();
16603        // Formula cells under `rect` on `sheet`: identity runs per column.
16604        let push_formulas = |stack: &mut Vec<(Cell, u32)>,
16605                             visited: &FxHashSet<Cell>,
16606                             sheet: u16,
16607                             rect: Rect| {
16608            for col in rect.c0..=rect.c1 {
16609                ids.visit_runs_in(sheet, col, rect.r0, rect.r1, &mut |h| {
16610                    let run = ids.run(h);
16611                    let r0 = run.row_start.max(rect.r0);
16612                    let r1 = (run.row_start + run.len - 1).min(rect.r1);
16613                    for row in r0..=r1 {
16614                        if !visited.contains(&(sheet, row, col)) {
16615                            stack.push(((sheet, row, col), run.first_id + (row - run.row_start)));
16616                        }
16617                    }
16618                });
16619            }
16620        };
16621        for &v in targets {
16622            if let Some(table) = self.graph.table_by_vertex(v) {
16623                // A table's demand is its range's, as legacy's table vertex
16624                // leads to the cells it covers (its symbol row has no
16625                // precedents).
16626                let (s, e) = (table.range.start, table.range.end);
16627                push_formulas(
16628                    &mut stack,
16629                    &visited,
16630                    s.sheet_id,
16631                    Rect::new(s.coord.row(), s.coord.col(), e.coord.row(), e.coord.col()),
16632                );
16633            } else if let Some(cell) = self.graph.authority_cell_of_vertex(v) {
16634                stack.push((cell, crate::engine::authority::identity::NO_VID));
16635            }
16636        }
16637        let mut hits = Vec::new();
16638        #[cfg(any(test, feature = "benchmark_internal"))]
16639        let (mut probe_vertices, mut probe_clean_formulas, mut probe_edges, mut probe_dynamic) =
16640            (0, 0, 0, 0);
16641        while let Some((cell, id)) = stack.pop() {
16642            if !visited.insert(cell) {
16643                continue;
16644            }
16645            let Some(v) = self.graph.authority_vertex_of_formula(id, cell) else {
16646                continue;
16647            };
16648            if !self.graph.vertex_exists(v) {
16649                continue;
16650            }
16651            #[cfg(any(test, feature = "benchmark_internal"))]
16652            {
16653                probe_vertices += 1;
16654            }
16655            match self.graph.get_vertex_kind(v) {
16656                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
16657                    if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
16658                        to_evaluate.insert(v);
16659                    } else {
16660                        #[cfg(any(test, feature = "benchmark_internal"))]
16661                        {
16662                            probe_clean_formulas += 1;
16663                        }
16664                    }
16665                }
16666                VertexKind::NamedScalar | VertexKind::NamedArray => {
16667                    to_evaluate.insert(v);
16668                }
16669                _ => {}
16670            }
16671            hits.clear();
16672            store.direct_precedents(cell, TagFilter::All, &mut hits);
16673            #[cfg(any(test, feature = "benchmark_internal"))]
16674            {
16675                probe_edges += hits.len();
16676            }
16677            for &(_, sheet, rect) in &hits {
16678                // DirtyExtents: a dirty spill anchor whose extent meets this
16679                // image is a demand precedent, ordered before `v` (§8.2).
16680                if sheet != SYMBOL_SHEET {
16681                    for anchor in self
16682                        .graph
16683                        .spill_anchors_in_region(sheet, rect.r0, rect.c0, rect.r1, rect.c1)
16684                    {
16685                        if anchor != v && self.graph.is_dirty(anchor) {
16686                            vdeps.entry(v).or_default().push(anchor);
16687                            if let Some(c) = self.graph.authority_cell_of_vertex(anchor) {
16688                                stack.push((c, crate::engine::authority::identity::NO_VID));
16689                            }
16690                        }
16691                    }
16692                }
16693                if sheet == SYMBOL_SHEET {
16694                    stack.extend((rect.r0..=rect.r1).map(|slot| {
16695                        (
16696                            (SYMBOL_SHEET, slot, 0),
16697                            crate::engine::authority::identity::NO_VID,
16698                        )
16699                    }));
16700                    continue;
16701                }
16702                push_formulas(&mut stack, &visited, sheet, rect);
16703            }
16704            if self.graph.is_dynamic(v) {
16705                #[cfg(any(test, feature = "benchmark_internal"))]
16706                {
16707                    probe_dynamic += 1;
16708                }
16709                // rdi_dyn: the observed reads are demand precedents.
16710                if let Some(reads) = self.graph.authority_host().observed(v) {
16711                    for &(sheet, r0, c0, r1, c1) in reads {
16712                        push_formulas(&mut stack, &visited, sheet, Rect::new(r0, c0, r1, c1));
16713                    }
16714                }
16715                let (vdeps_map, _) = VirtualDepBuilder::new(self).build(&[v]);
16716                // Pre-probe targets plus reads this request found dirty
16717                // (design §8.3: demand walks rdi ∪ hints).
16718                let hinted = self.freshness_hints(v).unwrap_or(&[]);
16719                if let Some(deps) = vdeps_map
16720                    .get(&v)
16721                    .map(|deps| deps.iter().chain(hinted))
16722                    .or(Some([].iter().chain(hinted)))
16723                {
16724                    for &u in deps {
16725                        vdeps.entry(v).or_default().push(u);
16726                        if let Some(c) = self.graph.authority_cell_of_vertex(u) {
16727                            stack.push((c, crate::engine::authority::identity::NO_VID));
16728                        }
16729                    }
16730                }
16731            }
16732        }
16733        let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
16734        result.sort_unstable();
16735        for deps in vdeps.values_mut() {
16736            deps.sort_unstable();
16737            deps.dedup();
16738        }
16739        #[cfg(any(test, feature = "benchmark_internal"))]
16740        {
16741            let mut probe = self.recalc_reuse_probe.lock().unwrap();
16742            probe.demand_builds += 1;
16743            probe.demand_vertices += probe_vertices;
16744            probe.demand_clean_formulas += probe_clean_formulas;
16745            probe.demand_explicit_edges += probe_edges;
16746            probe.demand_virtual_builder_calls += probe_dynamic;
16747        }
16748        Ok((result, vdeps))
16749    }
16750
16751    /// Helper: convert 1-based column index to Excel-style letters (1 -> A, 27 -> AA)
16752    fn col_to_letters(col: u32) -> String {
16753        col_letters_from_1based(col).expect("column index must be >= 1")
16754    }
16755
16756    /// Evaluate all dirty/volatile vertices with cancellation support
16757    pub fn evaluate_all_cancellable(
16758        &mut self,
16759        cancel: crate::engine::CancelToken,
16760    ) -> Result<EvalResult, ExcelError> {
16761        self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
16762            engine.observe_function_semantic_epoch()?;
16763            engine.active_cancel_flag = Some(cancel.clone());
16764            let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
16765            engine.active_cancel_flag = None;
16766            res
16767        })
16768    }
16769
16770    fn evaluate_all_cancellable_impl(
16771        &mut self,
16772        cancel_flag: &AtomicBool,
16773    ) -> Result<EvalResult, ExcelError> {
16774        let _source_cache = self.source_cache_session();
16775        self.validate_deterministic_mode()?;
16776        if self.config.defer_graph_building {
16777            self.build_graph_all()?;
16778        }
16779        if cancel_flag.load(Ordering::Relaxed) {
16780            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
16781                .with_message("Evaluation cancelled before scheduling".to_string()));
16782        }
16783        self.require_unified_authority()?;
16784        self.begin_evaluation_request();
16785        self.reset_virtual_dep_telemetry_if_disabled();
16786        let start = crate::instant::FzInstant::now();
16787        let mut computed_vertices = 0;
16788        let mut cycle_errors = 0;
16789
16790        let mut replan_iterations = 0;
16791        const MAX_REPLAN: usize = 5;
16792        let mut telemetry = self
16793            .config
16794            .enable_virtual_dep_telemetry
16795            .then(|| self.start_virtual_dep_telemetry());
16796
16797        loop {
16798            if cancel_flag.load(Ordering::Relaxed) {
16799                if let Some(mut t) = telemetry {
16800                    t.bailout_reason = Some("cancelled");
16801                    t.replan_iterations = replan_iterations;
16802                    self.last_virtual_dep_telemetry = t;
16803                }
16804                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
16805                    .with_message("Evaluation cancelled before scheduling".to_string()));
16806            }
16807
16808            let to_evaluate = self.graph.get_evaluation_vertices();
16809            if to_evaluate.is_empty() {
16810                if let Some(t) = telemetry.as_mut()
16811                    && t.bailout_reason.is_none()
16812                {
16813                    t.bailout_reason = Some("no_work");
16814                }
16815                break;
16816            }
16817
16818            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
16819            if let Some(t) = telemetry.as_mut() {
16820                Self::accumulate_schedule_meta(t, &meta);
16821            }
16822
16823            // Walk units in condensation order, checking cancellation between
16824            // units (formerly between cycles and between layers).
16825            self.begin_pass(&schedule);
16826            for (unit_index, &unit) in schedule.units.iter().enumerate() {
16827                match unit {
16828                    ScheduleUnit::Cycle(i) => {
16829                        // Check cancellation between cycles
16830                        if cancel_flag.load(Ordering::Relaxed) {
16831                            if let Some(mut t) = telemetry {
16832                                t.bailout_reason = Some("cancelled");
16833                                t.replan_iterations = replan_iterations;
16834                                self.last_virtual_dep_telemetry = t;
16835                            }
16836                            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
16837                                "Evaluation cancelled during cycle handling".to_string(),
16838                            ));
16839                        }
16840
16841                        if self.handle_cycle_unit(
16842                            schedule.unit_cycle(i),
16843                            None,
16844                            None,
16845                            Some(cancel_flag),
16846                        )? > 0
16847                        {
16848                            cycle_errors += 1;
16849                        }
16850                    }
16851                    ScheduleUnit::Layer(i) => {
16852                        let layer = schedule.unit_layer(i);
16853                        // Check cancellation between layers
16854                        if cancel_flag.load(Ordering::Relaxed) {
16855                            if let Some(mut t) = telemetry {
16856                                t.bailout_reason = Some("cancelled");
16857                                t.replan_iterations = replan_iterations;
16858                                self.last_virtual_dep_telemetry = t;
16859                            }
16860                            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
16861                                .with_message("Evaluation cancelled between layers".to_string()));
16862                        }
16863
16864                        // Evaluate vertices in this layer (parallel or sequential)
16865                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
16866                            computed_vertices +=
16867                                self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
16868                        } else {
16869                            computed_vertices +=
16870                                self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
16871                        }
16872                    }
16873                }
16874                if self.stop_after_unit(&schedule, unit_index) {
16875                    break;
16876                }
16877            }
16878
16879            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
16880            if let Some(t) = telemetry.as_mut() {
16881                t.changed_vdeps_total += changed_vertices.len();
16882            }
16883            self.resource_checkpoint(0)?;
16884            if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
16885                if let Some(t) = telemetry.as_mut() {
16886                    t.bailout_reason = Some("converged");
16887                }
16888                break;
16889            }
16890            if replan_iterations >= MAX_REPLAN {
16891                if let Some(mut t) = telemetry.take() {
16892                    t.bailout_reason = Some("max_replan");
16893                    t.replan_iterations = replan_iterations;
16894                    self.last_virtual_dep_telemetry = t;
16895                }
16896                return Err(
16897                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
16898                );
16899            }
16900            replan_iterations += 1;
16901        }
16902
16903        if let Some(mut t) = telemetry {
16904            t.replan_iterations = replan_iterations;
16905            self.last_virtual_dep_telemetry = t;
16906        }
16907
16908        // Re-dirty volatile vertices for the next evaluation cycle
16909        self.redirty_for_next_recalc();
16910        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
16911
16912        Ok(EvalResult {
16913            computed_vertices,
16914            cycle_errors,
16915            elapsed: start.elapsed(),
16916        })
16917    }
16918
16919    /// Evaluate only the necessary precedents for specific target cells with cancellation support
16920    pub fn evaluate_until_cancellable(
16921        &mut self,
16922        targets: &[&str],
16923        cancel: crate::engine::CancelToken,
16924    ) -> Result<EvalResult, ExcelError> {
16925        self.observe_evaluation_resource_request(
16926            EvaluationRequestKind::TargetedCancellable,
16927            |engine| {
16928                engine.observe_function_semantic_epoch()?;
16929                engine.active_cancel_flag = Some(cancel.clone());
16930                let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
16931                engine.active_cancel_flag = None;
16932                res
16933            },
16934        )
16935    }
16936
16937    fn evaluate_until_cancellable_impl(
16938        &mut self,
16939        targets: &[&str],
16940        cancel_flag: &AtomicBool,
16941    ) -> Result<EvalResult, ExcelError> {
16942        if cancel_flag.load(Ordering::Relaxed) {
16943            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
16944                .with_message("Evaluation cancelled before target preparation"));
16945        }
16946        let mut typed_targets = Vec::with_capacity(targets.len());
16947        for target in targets {
16948            let (sheet, row, col) = self.parse_a1_notation(target)?;
16949            self.graph.sheet_id_mut(&sheet);
16950            typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
16951        }
16952        self.evaluate_mixed_targets(&typed_targets, None)
16953    }
16954
16955    fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
16956        let mut quoted = false;
16957        let mut separator = None;
16958        let bytes = address.as_bytes();
16959        let mut index = 0usize;
16960        while index < bytes.len() {
16961            match bytes[index] {
16962                b'\'' => {
16963                    if quoted && bytes.get(index + 1) == Some(&b'\'') {
16964                        index = index.saturating_add(1);
16965                    } else {
16966                        quoted = !quoted;
16967                    }
16968                }
16969                b'!' if !quoted => separator = Some(index),
16970                _ => {}
16971            }
16972            index = index.saturating_add(1);
16973        }
16974        if quoted {
16975            return Err(ExcelError::new(ExcelErrorKind::Ref)
16976                .with_message(format!("Invalid quoted sheet reference `{address}`")));
16977        }
16978        let (sheet, cell_part) = match separator {
16979            Some(separator) => {
16980                let raw_sheet = &address[..separator];
16981                let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
16982                    raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
16983                } else {
16984                    raw_sheet.to_string()
16985                };
16986                (sheet, &address[separator + 1..])
16987            }
16988            None => (self.default_sheet_name().to_string(), address),
16989        };
16990
16991        let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
16992            ExcelError::new(ExcelErrorKind::Ref)
16993                .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
16994        })?;
16995
16996        Ok((sheet, row, col))
16997    }
16998
16999    /// Determine volatility using this engine's FunctionProvider, falling back to global registry.
17000    fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
17001        use formualizer_parse::parser::ASTNodeType;
17002        match &ast.node_type {
17003            ASTNodeType::Function { name, args, .. } => {
17004                if let Some(func) = self
17005                    .get_function("", name)
17006                    .or_else(|| crate::function_registry::get("", name))
17007                    && func.caps().contains(crate::function::FnCaps::VOLATILE)
17008                {
17009                    return true;
17010                }
17011                args.iter()
17012                    .any(|arg| self.is_ast_volatile_with_provider(arg))
17013            }
17014            ASTNodeType::BinaryOp { left, right, .. } => {
17015                self.is_ast_volatile_with_provider(left)
17016                    || self.is_ast_volatile_with_provider(right)
17017            }
17018            ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
17019            ASTNodeType::Array(rows) => rows.iter().any(|row| {
17020                row.iter()
17021                    .any(|cell| self.is_ast_volatile_with_provider(cell))
17022            }),
17023            _ => false,
17024        }
17025    }
17026
17027    /// Evaluate a layer sequentially
17028    fn evaluate_layer_sequential(
17029        &mut self,
17030        layer: &super::scheduler::Layer,
17031    ) -> Result<usize, ExcelError> {
17032        self.resource_checkpoint(layer.vertices.len() as u64)?;
17033        self.evaluate_layer_sequential_effects(layer)
17034    }
17035
17036    fn update_vertex_value_with_delta(
17037        &mut self,
17038        vertex_id: VertexId,
17039        new_value: LiteralValue,
17040        delta: &mut DeltaCollector,
17041    ) {
17042        if delta.mode != DeltaMode::Off
17043            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
17044        {
17045            let sheet_name = self.graph.sheet_name(cell.sheet_id);
17046            let old = self
17047                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
17048                .unwrap_or(LiteralValue::Empty);
17049            if old != new_value {
17050                delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17051            }
17052        }
17053        self.graph.update_vertex_value_ref(vertex_id, &new_value);
17054        self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
17055    }
17056
17057    fn evaluate_layer_sequential_with_delta(
17058        &mut self,
17059        layer: &super::scheduler::Layer,
17060        delta: &mut DeltaCollector,
17061    ) -> Result<usize, ExcelError> {
17062        self.resource_checkpoint(layer.vertices.len() as u64)?;
17063        self.evaluate_layer_sequential_with_delta_effects(layer, delta)
17064    }
17065
17066    /// Evaluate a layer sequentially with cancellation support
17067    fn evaluate_layer_sequential_cancellable(
17068        &mut self,
17069        layer: &super::scheduler::Layer,
17070        cancel_flag: &AtomicBool,
17071    ) -> Result<usize, ExcelError> {
17072        self.resource_checkpoint(layer.vertices.len() as u64)?;
17073        self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
17074    }
17075
17076    /// Evaluate a layer sequentially with more frequent cancellation checks for demand-driven evaluation
17077    fn evaluate_layer_sequential_cancellable_demand_driven(
17078        &mut self,
17079        layer: &super::scheduler::Layer,
17080        cancel_flag: &AtomicBool,
17081    ) -> Result<usize, ExcelError> {
17082        self.resource_checkpoint(layer.vertices.len() as u64)?;
17083        self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
17084    }
17085
17086    /// Evaluate a layer in parallel using the thread pool.
17087    ///
17088    /// Cost-adaptive: the layer starts sequentially in slices of doubling
17089    /// size (1, 2, 4, ... vertices, capped so a slice does not overshoot the
17090    /// probe) and hands the rest to the pool once the rest looks worth it
17091    /// (`PARALLEL_LAYER_WORTH` at the rate so far) or the probe
17092    /// (`PARALLEL_LAYER_PROBE`) is spent. A cheap layer never pays the pool's
17093    /// wake-up and join (most Enron layers are tens of µs of work); an
17094    /// expensive one goes parallel after a few vertices. Splitting a layer
17095    /// into consecutive sub-layers is a valid order: its vertices are
17096    /// independent.
17097    fn evaluate_layer_parallel(
17098        &mut self,
17099        layer: &super::scheduler::Layer,
17100    ) -> Result<usize, ExcelError> {
17101        if layer.sequential {
17102            return self.evaluate_layer_sequential(layer);
17103        }
17104        self.resource_checkpoint(layer.vertices.len() as u64)?;
17105        let len = layer.vertices.len();
17106        let buffered = buffer_layer_writes(layer);
17107        let (probe, worth) = (PARALLEL_LAYER_PROBE, PARALLEL_LAYER_WORTH);
17108        let start = crate::instant::FzInstant::now();
17109        let mut pos = 0usize;
17110        let mut step = 1usize;
17111        while pos < len {
17112            if pos > 0 {
17113                let elapsed = start.elapsed();
17114                // Rate so far (ns per vertex) and the rest at that rate.
17115                let per_vertex = elapsed.as_nanos() / pos as u128 + 1;
17116                let rest_estimate = per_vertex * (len - pos) as u128;
17117                if len - pos >= 2 && (elapsed >= probe || rest_estimate >= worth.as_nanos()) {
17118                    let rest = layer.sub_layer(pos, len);
17119                    // Expensive members (a SUMIF over a table) parallelize
17120                    // one per task; cheap ones keep runs of 8 together.
17121                    let min_chunk = if per_vertex >= EXPENSIVE_VERTEX_NS {
17122                        1
17123                    } else {
17124                        8
17125                    };
17126                    return Ok(pos + self.evaluate_layer_parallel_effects(&rest, min_chunk)?);
17127                }
17128                // Next slice: double, but no more than the rest of the probe
17129                // at the rate so far (a slice must not overshoot it).
17130                let fit = (probe.saturating_sub(elapsed).as_nanos() / per_vertex) as usize + 1;
17131                step = step.saturating_mul(2).min(fit);
17132            }
17133            let end = (pos + step).min(len);
17134            let slice = layer.sub_layer(pos, end);
17135            // A slice stops at the probe's end even if its members turn out
17136            // far more expensive than the rate so far predicted.
17137            pos += self.evaluate_layer_units_until(
17138                &slice,
17139                None,
17140                None,
17141                None,
17142                buffered,
17143                Some(start + probe),
17144            )?;
17145        }
17146        Ok(len)
17147    }
17148
17149    fn evaluate_layer_parallel_with_delta(
17150        &mut self,
17151        layer: &super::scheduler::Layer,
17152        delta: &mut DeltaCollector,
17153    ) -> Result<usize, ExcelError> {
17154        if layer.sequential {
17155            return self.evaluate_layer_sequential_with_delta(layer, delta);
17156        }
17157        self.resource_checkpoint(layer.vertices.len() as u64)?;
17158        self.evaluate_layer_parallel_with_delta_effects(layer, delta)
17159    }
17160
17161    /// Evaluate a layer in parallel with cancellation support
17162    fn evaluate_layer_parallel_cancellable(
17163        &mut self,
17164        layer: &super::scheduler::Layer,
17165        cancel_flag: &AtomicBool,
17166    ) -> Result<usize, ExcelError> {
17167        if layer.sequential {
17168            return self.evaluate_layer_sequential_cancellable(layer, cancel_flag);
17169        }
17170        self.resource_checkpoint(layer.vertices.len() as u64)?;
17171        self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
17172    }
17173
17174    /// Evaluate a single vertex without mutating the graph (for parallel evaluation)
17175    fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17176        // Check if vertex exists
17177        if !self.graph.vertex_exists(vertex_id) {
17178            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17179                .with_message(format!("Vertex not found: {vertex_id:?}")));
17180        }
17181
17182        // Get vertex kind and check if it needs evaluation
17183        let kind = self.graph.get_vertex_kind(vertex_id);
17184        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17185
17186        let view = match kind {
17187            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17188                if let Some(view) = self.graph.formula_view(vertex_id) {
17189                    view
17190                } else {
17191                    return Ok(LiteralValue::Number(0.0));
17192                }
17193            }
17194            VertexKind::Empty | VertexKind::Cell => {
17195                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17196                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17197                    let row = cell_ref.coord.row() + 1;
17198                    let col = cell_ref.coord.col() + 1;
17199                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17200                        return Ok(v);
17201                    }
17202                }
17203                return Ok(LiteralValue::Number(0.0));
17204            }
17205            VertexKind::NamedScalar => {
17206                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17207                    ExcelError::new(ExcelErrorKind::Name)
17208                        .with_message("Named range metadata missing".to_string())
17209                })?;
17210
17211                return match &named_range.definition {
17212                    NamedDefinition::Cell(cell_ref) => {
17213                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17214                        Ok(self
17215                            .get_cell_value(
17216                                sheet_name,
17217                                cell_ref.coord.row() + 1,
17218                                cell_ref.coord.col() + 1,
17219                            )
17220                            .unwrap_or(LiteralValue::Empty))
17221                    }
17222                    NamedDefinition::Literal(v) => Ok(v.clone()),
17223                    NamedDefinition::Formula { ast, .. } => {
17224                        let context_sheet = match named_range.scope {
17225                            NameScope::Sheet(id) => id,
17226                            NameScope::Workbook => sheet_id,
17227                        };
17228                        let sheet_name = self.graph.sheet_name(context_sheet);
17229                        let cell_ref = self
17230                            .graph
17231                            .get_cell_ref(vertex_id)
17232                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17233                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17234                        interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
17235                    }
17236                    NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17237                        .with_message("Range-valued name evaluated as scalar".to_string())),
17238                };
17239            }
17240            VertexKind::NamedArray => {
17241                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17242                    ExcelError::new(ExcelErrorKind::Name)
17243                        .with_message("Named range metadata missing".to_string())
17244                })?;
17245
17246                return match &named_range.definition {
17247                    NamedDefinition::Range(range_ref) => {
17248                        if range_ref.start.sheet_id != range_ref.end.sheet_id {
17249                            return Err(ExcelError::new(ExcelErrorKind::Ref)
17250                                .with_message("Named range cannot span sheets".to_string()));
17251                        }
17252                        let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17253                        let sr0 = range_ref.start.coord.row();
17254                        let sc0 = range_ref.start.coord.col();
17255                        let er0 = range_ref.end.coord.row();
17256                        let ec0 = range_ref.end.coord.col();
17257                        if sr0 > er0 || sc0 > ec0 {
17258                            return Err(ExcelError::new(ExcelErrorKind::Ref)
17259                                .with_message("Invalid named range bounds".to_string()));
17260                        }
17261
17262                        let h = (er0 - sr0 + 1) as usize;
17263                        let w = (ec0 - sc0 + 1) as usize;
17264                        let cell_count = (h as u64).saturating_mul(w as u64);
17265                        if cell_count > self.config.spill.max_spill_cells as u64 {
17266                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17267                                "Named range too large to materialize as an array".to_string(),
17268                            ));
17269                        }
17270
17271                        // `get_cell_value` per cell, with the sheet resolved
17272                        // once (`read_cell_formatted_in` is its body).
17273                        let sheet_id = range_ref.start.sheet_id;
17274                        let asheet = self.arrow_sheets.sheet(sheet_name);
17275                        let mut rows = Vec::with_capacity(h);
17276                        for r0 in sr0..=er0 {
17277                            let mut row = Vec::with_capacity(w);
17278                            for c0 in sc0..=ec0 {
17279                                row.push(
17280                                    self.read_cell_formatted_in(sheet_id, asheet, r0 + 1, c0 + 1)
17281                                        .0,
17282                                );
17283                            }
17284                            rows.push(row);
17285                        }
17286                        Ok(LiteralValue::Array(rows))
17287                    }
17288                    NamedDefinition::Cell(cell_ref) => {
17289                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17290                        let row = cell_ref.coord.row() + 1;
17291                        let col = cell_ref.coord.col() + 1;
17292                        let v = self
17293                            .get_cell_value(sheet_name, row, col)
17294                            .unwrap_or(LiteralValue::Empty);
17295                        Ok(LiteralValue::Array(vec![vec![v]]))
17296                    }
17297                    NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
17298                    NamedDefinition::Formula { ast, .. } => {
17299                        let context_sheet = match named_range.scope {
17300                            NameScope::Sheet(id) => id,
17301                            NameScope::Workbook => sheet_id,
17302                        };
17303                        let sheet_name = self.graph.sheet_name(context_sheet);
17304                        let cell_ref = self
17305                            .graph
17306                            .get_cell_ref(vertex_id)
17307                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17308                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17309                        match interpreter.evaluate_ast(ast) {
17310                            Ok(cv) => {
17311                                let v = cv.into_literal();
17312                                match v {
17313                                    LiteralValue::Array(_) => Ok(v),
17314                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
17315                                }
17316                            }
17317                            Err(err) => Ok(LiteralValue::Error(err)),
17318                        }
17319                    }
17320                };
17321            }
17322            VertexKind::InfiniteRange
17323            | VertexKind::Range
17324            | VertexKind::External
17325            | VertexKind::Table => {
17326                // Not directly evaluatable here.
17327                return Ok(LiteralValue::Number(0.0));
17328            }
17329        };
17330
17331        // The interpreter uses a reference to the engine as the context
17332        let sheet_name = self.graph.sheet_name(sheet_id);
17333        let cell_ref = self
17334            .graph
17335            .get_cell_ref(vertex_id)
17336            .expect("cell ref for vertex");
17337        if let Some(result) =
17338            self.freshness_evaluate_recorded(vertex_id, sheet_name, cell_ref, view)
17339        {
17340            return result;
17341        }
17342        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17343
17344        interpreter
17345            .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
17346            .map(|cv| {
17347                let format = cv.format_id();
17348                self.record_derived_format(vertex_id, format);
17349                self.materialize_formula_result(vertex_id, cv)
17350            })
17351            .or_else(|error| self.fit_formula_error(vertex_id, error))
17352    }
17353
17354    /// Get access to the shared thread pool for parallel evaluation
17355    pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
17356        self.thread_pool.as_ref()
17357    }
17358}
17359
17360#[derive(Default)]
17361struct RowBoundsCache {
17362    snapshot: u64,
17363    // key: (sheet_id, col_idx)
17364    map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
17365}
17366
17367impl RowBoundsCache {
17368    fn new(snapshot: u64) -> Self {
17369        Self {
17370            snapshot,
17371            map: Default::default(),
17372        }
17373    }
17374    fn get_row_bounds(
17375        &self,
17376        sheet_id: SheetId,
17377        col_idx: usize,
17378        snapshot: u64,
17379    ) -> Option<(Option<u32>, Option<u32>)> {
17380        if self.snapshot != snapshot {
17381            return None;
17382        }
17383        self.map.get(&(sheet_id as u32, col_idx)).copied()
17384    }
17385    fn put_row_bounds(
17386        &mut self,
17387        sheet_id: SheetId,
17388        col_idx: usize,
17389        snapshot: u64,
17390        bounds: (Option<u32>, Option<u32>),
17391    ) {
17392        if self.snapshot != snapshot {
17393            self.snapshot = snapshot;
17394            self.map.clear();
17395        }
17396        self.map.insert((sheet_id as u32, col_idx), bounds);
17397    }
17398}
17399
17400struct UsedAxisBoundsCache {
17401    snapshot: u64,
17402    row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17403    col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17404    #[cfg(test)]
17405    row_hits: std::sync::atomic::AtomicUsize,
17406    #[cfg(test)]
17407    row_misses: std::sync::atomic::AtomicUsize,
17408    #[cfg(test)]
17409    col_hits: std::sync::atomic::AtomicUsize,
17410    #[cfg(test)]
17411    col_misses: std::sync::atomic::AtomicUsize,
17412}
17413
17414impl UsedAxisBoundsCache {
17415    fn new(snapshot: u64) -> Self {
17416        Self {
17417            snapshot,
17418            row_bounds_by_col_span: Default::default(),
17419            col_bounds_by_row_span: Default::default(),
17420            #[cfg(test)]
17421            row_hits: std::sync::atomic::AtomicUsize::new(0),
17422            #[cfg(test)]
17423            row_misses: std::sync::atomic::AtomicUsize::new(0),
17424            #[cfg(test)]
17425            col_hits: std::sync::atomic::AtomicUsize::new(0),
17426            #[cfg(test)]
17427            col_misses: std::sync::atomic::AtomicUsize::new(0),
17428        }
17429    }
17430
17431    fn reset_for_snapshot(&mut self, snapshot: u64) {
17432        if self.snapshot != snapshot {
17433            self.snapshot = snapshot;
17434            self.row_bounds_by_col_span.clear();
17435            self.col_bounds_by_row_span.clear();
17436        }
17437    }
17438
17439    fn get_row_bounds(
17440        &self,
17441        sheet_id: SheetId,
17442        start_col: u32,
17443        end_col: u32,
17444        snapshot: u64,
17445    ) -> Option<Option<(u32, u32)>> {
17446        if self.snapshot != snapshot {
17447            return None;
17448        }
17449        let cached = self
17450            .row_bounds_by_col_span
17451            .get(&(sheet_id, start_col, end_col))
17452            .copied();
17453        #[cfg(test)]
17454        if cached.is_some() {
17455            self.row_hits.fetch_add(1, Ordering::Relaxed);
17456        }
17457        cached
17458    }
17459
17460    fn put_row_bounds(
17461        &mut self,
17462        sheet_id: SheetId,
17463        start_col: u32,
17464        end_col: u32,
17465        snapshot: u64,
17466        bounds: Option<(u32, u32)>,
17467    ) {
17468        self.reset_for_snapshot(snapshot);
17469        self.row_bounds_by_col_span
17470            .insert((sheet_id, start_col, end_col), bounds);
17471        #[cfg(test)]
17472        self.row_misses.fetch_add(1, Ordering::Relaxed);
17473    }
17474
17475    fn get_col_bounds(
17476        &self,
17477        sheet_id: SheetId,
17478        start_row: u32,
17479        end_row: u32,
17480        snapshot: u64,
17481    ) -> Option<Option<(u32, u32)>> {
17482        if self.snapshot != snapshot {
17483            return None;
17484        }
17485        let cached = self
17486            .col_bounds_by_row_span
17487            .get(&(sheet_id, start_row, end_row))
17488            .copied();
17489        #[cfg(test)]
17490        if cached.is_some() {
17491            self.col_hits.fetch_add(1, Ordering::Relaxed);
17492        }
17493        cached
17494    }
17495
17496    fn put_col_bounds(
17497        &mut self,
17498        sheet_id: SheetId,
17499        start_row: u32,
17500        end_row: u32,
17501        snapshot: u64,
17502        bounds: Option<(u32, u32)>,
17503    ) {
17504        self.reset_for_snapshot(snapshot);
17505        self.col_bounds_by_row_span
17506            .insert((sheet_id, start_row, end_row), bounds);
17507        #[cfg(test)]
17508        self.col_misses.fetch_add(1, Ordering::Relaxed);
17509    }
17510}
17511
17512// Phase 2 shim: in-process spill manager delegating to current graph methods.
17513#[derive(Default)]
17514pub struct ShimSpillManager {
17515    region_locks: RegionLockManager,
17516    pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
17517}
17518
17519impl ShimSpillManager {
17520    pub(crate) fn reserve(
17521        &mut self,
17522        owner: VertexId,
17523        anchor_cell: CellRef,
17524        shape: SpillShape,
17525        _meta: SpillMeta,
17526    ) -> Result<(), ExcelError> {
17527        // Derive region from anchor + shape; enforce in-flight exclusivity only.
17528        let region = crate::engine::spill::Region {
17529            sheet_id: anchor_cell.sheet_id as u32,
17530            row_start: anchor_cell.coord.row(),
17531            row_end: anchor_cell
17532                .coord
17533                .row()
17534                .saturating_add(shape.rows)
17535                .saturating_sub(1),
17536            col_start: anchor_cell.coord.col(),
17537            col_end: anchor_cell
17538                .coord
17539                .col()
17540                .saturating_add(shape.cols)
17541                .saturating_sub(1),
17542        };
17543        match self.region_locks.reserve(region, owner) {
17544            Ok(id) => {
17545                if id != 0 {
17546                    self.active_locks.insert(owner, id);
17547                }
17548                Ok(())
17549            }
17550            Err(e) => Err(e),
17551        }
17552    }
17553
17554    /// Release any in-flight region reservation still held for `owner`.
17555    ///
17556    /// Reservations are normally released on commit/rollback, but if an anchor is
17557    /// abandoned without committing (e.g. cycle detection stamps it with #CIRC), a
17558    /// stale reservation could remain. This is a no-op when nothing is held.
17559    pub(crate) fn release_owner(&mut self, owner: VertexId) {
17560        if let Some(id) = self.active_locks.remove(&owner) {
17561            self.region_locks.release(id);
17562        }
17563    }
17564
17565    pub(crate) fn commit_array_with_value_probe<F>(
17566        &mut self,
17567        graph: &mut DependencyGraph,
17568        anchor_vertex: VertexId,
17569        targets: &[CellRef],
17570        rows: Vec<Vec<LiteralValue>>,
17571        mut value_probe: F,
17572    ) -> Result<(), ExcelError>
17573    where
17574        F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
17575    {
17576        use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
17577
17578        // Re-run plan on concrete targets before committing to respect blockers.
17579        // This plan checks formula/spill ownership in the graph, but when the graph value cache
17580        // is disabled (Arrow-canonical mode), it cannot see non-empty value blockers.
17581        let plan_res = graph.plan_spill_region(anchor_vertex, targets);
17582        if let Err(e) = plan_res {
17583            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17584                self.region_locks.release(id);
17585            }
17586            return Err(e);
17587        }
17588
17589        if !graph.value_cache_enabled() {
17590            // Compute expected spill shape from the target rectangle for diagnostics.
17591            let (expected_rows, expected_cols) = if targets.is_empty() {
17592                (0u32, 0u32)
17593            } else {
17594                let mut min_r = u32::MAX;
17595                let mut max_r = 0u32;
17596                let mut min_c = u32::MAX;
17597                let mut max_c = 0u32;
17598                for cell in targets {
17599                    let r = cell.coord.row();
17600                    let c = cell.coord.col();
17601                    min_r = min_r.min(r);
17602                    max_r = max_r.max(r);
17603                    min_c = min_c.min(c);
17604                    max_c = max_c.max(c);
17605                }
17606                (
17607                    max_r.saturating_sub(min_r).saturating_add(1),
17608                    max_c.saturating_sub(min_c).saturating_add(1),
17609                )
17610            };
17611
17612            let anchor_cell = graph
17613                .get_cell_ref(anchor_vertex)
17614                .expect("anchor cell ref for spill commit");
17615
17616            for cell in targets {
17617                // Never treat the anchor as a blocker.
17618                if *cell == anchor_cell {
17619                    continue;
17620                }
17621                // Skip cells already known to be owned by a spill; plan() handled spill conflicts.
17622                if graph.spill_registry_anchor_for_cell(*cell).is_some() {
17623                    continue;
17624                }
17625                // Skip formula vertices in the target region; plan() handled them (or allowed).
17626                if let Some(vid) = graph.get_vertex_id_for_address(cell)
17627                    && vid != anchor_vertex
17628                {
17629                    match graph.get_vertex_kind(vid) {
17630                        crate::engine::vertex::VertexKind::FormulaScalar
17631                        | crate::engine::vertex::VertexKind::FormulaArray => {
17632                            // plan() already approved allowed overwrites.
17633                            continue;
17634                        }
17635                        _ => {}
17636                    }
17637                }
17638
17639                if let Some(v) = value_probe(graph, cell)
17640                    && !matches!(v, LiteralValue::Empty)
17641                {
17642                    if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17643                        self.region_locks.release(id);
17644                    }
17645                    return Err(ExcelError::new(ExcelErrorKind::Spill)
17646                        .with_message("BlockedByValue")
17647                        .with_extra(ExcelErrorExtra::Spill {
17648                            expected_rows,
17649                            expected_cols,
17650                        }));
17651                }
17652            }
17653        }
17654
17655        let commit_res = graph.commit_spill_region_atomic_with_fault(
17656            anchor_vertex,
17657            targets.to_vec(),
17658            rows,
17659            None,
17660        );
17661        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17662            self.region_locks.release(id);
17663        }
17664        commit_res.map(|_| ())
17665    }
17666
17667    /// Commit a spill and mirror all written cells into Arrow overlay via the owning engine.
17668    pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
17669        &mut self,
17670        engine: &mut Engine<R>,
17671        anchor_vertex: VertexId,
17672        targets: &[CellRef],
17673        rows: Vec<Vec<LiteralValue>>,
17674    ) -> Result<(), ExcelError> {
17675        if let Err(error) = engine.guard_pending_spill_commit(anchor_vertex, targets) {
17676            self.release_owner(anchor_vertex);
17677            return Err(error);
17678        }
17679        // Re-run plan on concrete targets before committing to respect blockers.
17680        let plan_res = engine.graph.plan_spill_region(anchor_vertex, targets);
17681        if let Err(e) = plan_res {
17682            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17683                self.region_locks.release(id);
17684            }
17685            return Err(e);
17686        }
17687
17688        let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
17689            anchor_vertex,
17690            targets.to_vec(),
17691            rows.clone(),
17692            None,
17693        );
17694        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17695            self.region_locks.release(id);
17696        }
17697        commit_res.map(|_| ())?;
17698        engine
17699            .blocked_pending_spills
17700            .retain(|entry| entry.0 != anchor_vertex);
17701        engine.spill_blocker_waiters.forget(anchor_vertex);
17702
17703        // Mirror into Arrow overlay when enabled
17704        if engine.config.arrow_storage_enabled
17705            && engine.config.delta_overlay_enabled
17706            && engine.config.write_formula_overlay_enabled
17707        {
17708            // Expect targets to be a contiguous rectangle row-major starting at some anchor
17709            for (idx, cell) in targets.iter().enumerate() {
17710                let (r_off, c_off) = {
17711                    if rows.is_empty() || rows[0].is_empty() {
17712                        (0usize, 0usize)
17713                    } else {
17714                        let width = rows[0].len();
17715                        (idx / width, idx % width)
17716                    }
17717                };
17718                let v = rows
17719                    .get(r_off)
17720                    .and_then(|r| r.get(c_off))
17721                    .cloned()
17722                    .unwrap_or(LiteralValue::Empty);
17723                let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
17724                engine.mirror_value_to_computed_overlay(
17725                    &sheet_name,
17726                    cell.coord.row() + 1,
17727                    cell.coord.col() + 1,
17728                    &v,
17729                );
17730            }
17731        }
17732        Ok(())
17733    }
17734}
17735
17736impl<R> Engine<R>
17737where
17738    R: EvaluationContext,
17739{
17740    fn resolve_shared_ref(
17741        &self,
17742        reference: &ReferenceType,
17743        current_sheet: &str,
17744    ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
17745        use formualizer_common::{
17746            SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
17747            SheetRef as SharedRef,
17748        };
17749
17750        // Preserve anchor flags from the parsed reference when possible.
17751        let sr = match reference {
17752            ReferenceType::Cell {
17753                sheet,
17754                row,
17755                col,
17756                row_abs,
17757                col_abs,
17758            } => {
17759                let row0 = row
17760                    .checked_sub(1)
17761                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
17762                let col0 = col
17763                    .checked_sub(1)
17764                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
17765                let sheet_loc = match sheet.as_deref() {
17766                    Some(name) => SheetLocator::from_name(name),
17767                    None => SheetLocator::Current,
17768                };
17769                let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
17770                SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
17771            }
17772            ReferenceType::Range {
17773                sheet,
17774                start_row,
17775                start_col,
17776                end_row,
17777                end_col,
17778                start_row_abs,
17779                start_col_abs,
17780                end_row_abs,
17781                end_col_abs,
17782            } => {
17783                let sheet_loc = match sheet.as_deref() {
17784                    Some(name) => SheetLocator::from_name(name),
17785                    None => SheetLocator::Current,
17786                };
17787                let sr = start_row
17788                    .map(|r| {
17789                        r.checked_sub(1)
17790                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17791                    })
17792                    .transpose()?;
17793                let sc = start_col
17794                    .map(|c| {
17795                        c.checked_sub(1)
17796                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17797                    })
17798                    .transpose()?;
17799                let er = end_row
17800                    .map(|r| {
17801                        r.checked_sub(1)
17802                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17803                    })
17804                    .transpose()?;
17805                let ec = end_col
17806                    .map(|c| {
17807                        c.checked_sub(1)
17808                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17809                    })
17810                    .transpose()?;
17811                let range = SharedRangeRef::from_parts(
17812                    sheet_loc,
17813                    sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
17814                    sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
17815                    er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
17816                    ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
17817                )
17818                .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
17819                SharedRef::Range(range)
17820            }
17821            _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
17822        };
17823
17824        let current_id = self
17825            .graph
17826            .sheet_id(current_sheet)
17827            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
17828
17829        let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
17830            match loc {
17831                SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
17832                SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
17833                SheetLocator::Name(name) => {
17834                    let n = name.as_ref();
17835                    self.graph
17836                        .sheet_id(n)
17837                        .map(SheetLocator::Id)
17838                        .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17839                }
17840            }
17841        };
17842
17843        match sr {
17844            SharedRef::Cell(cell) => {
17845                let owned = cell.into_owned();
17846                let sheet = resolve_loc(owned.sheet)?;
17847                Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
17848            }
17849            SharedRef::Range(range) => {
17850                let owned = range.into_owned();
17851                let sheet = resolve_loc(owned.sheet)?;
17852                Ok(SharedRef::Range(SharedRangeRef {
17853                    sheet,
17854                    start_row: owned.start_row,
17855                    start_col: owned.start_col,
17856                    end_row: owned.end_row,
17857                    end_col: owned.end_col,
17858                }))
17859            }
17860        }
17861    }
17862}
17863
17864// Implement the resolver traits for the Engine.
17865// This allows the interpreter to resolve references by querying the engine's graph.
17866impl<R> crate::traits::ReferenceResolver for Engine<R>
17867where
17868    R: EvaluationContext,
17869{
17870    fn resolve_cell_reference(
17871        &self,
17872        sheet: Option<&str>,
17873        row: u32,
17874        col: u32,
17875    ) -> Result<LiteralValue, ExcelError> {
17876        // This context-free trait method has no knowledge of the formula's
17877        // current sheet, so an unqualified (`None`) reference cannot be resolved
17878        // here. Previously this fell back to `default_sheet_name()`, which leaked
17879        // the reference onto an unrelated sheet (issue #110). Interpreter paths
17880        // already qualify references with the current sheet before reaching this
17881        // method (see `Interpreter::implicit_intersection_from_reference`), and
17882        // the sheet-aware scalar path goes through `resolve_cell_reference_value`
17883        // with an explicit `current_sheet`. Returning #REF! for an unqualified
17884        // reference here surfaces the missing context instead of silently
17885        // returning data from the wrong sheet.
17886        let Some(sheet_name) = sheet else {
17887            return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
17888                "Unqualified cell reference resolved without sheet context".to_string(),
17889            ));
17890        };
17891        // Prefer engine's unified accessor which consults Arrow store for base values
17892        // and falls back to graph for formulas and stored values.
17893        if let Some(v) = self.get_cell_value(sheet_name, row, col) {
17894            Ok(v)
17895        } else {
17896            // Excel semantics: empty cell coerces to 0 in numeric contexts
17897            Ok(LiteralValue::Number(0.0))
17898        }
17899    }
17900}
17901
17902impl<R> crate::traits::RangeResolver for Engine<R>
17903where
17904    R: EvaluationContext,
17905{
17906    fn resolve_range_reference(
17907        &self,
17908        sheet: Option<&str>,
17909        sr: Option<u32>,
17910        sc: Option<u32>,
17911        er: Option<u32>,
17912        ec: Option<u32>,
17913    ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
17914        // For now, delegate range resolution to the external resolver.
17915        // A future optimization could be to handle this within the graph.
17916        self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
17917    }
17918}
17919
17920impl<R> crate::traits::NamedRangeResolver for Engine<R>
17921where
17922    R: EvaluationContext,
17923{
17924    fn resolve_named_range_reference(
17925        &self,
17926        name: &str,
17927    ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
17928        self.resolver.resolve_named_range_reference(name)
17929    }
17930}
17931
17932impl<R> crate::traits::TableResolver for Engine<R>
17933where
17934    R: EvaluationContext,
17935{
17936    fn resolve_table_reference(
17937        &self,
17938        tref: &formualizer_parse::parser::TableReference,
17939    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
17940        self.resolver.resolve_table_reference(tref)
17941    }
17942}
17943
17944impl<R> crate::traits::SourceResolver for Engine<R>
17945where
17946    R: EvaluationContext,
17947{
17948    fn source_scalar_version(&self, name: &str) -> Option<u64> {
17949        self.resolver.source_scalar_version(name)
17950    }
17951
17952    fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
17953        self.resolver.resolve_source_scalar(name)
17954    }
17955
17956    fn source_table_version(&self, name: &str) -> Option<u64> {
17957        self.resolver.source_table_version(name)
17958    }
17959
17960    fn resolve_source_table(
17961        &self,
17962        name: &str,
17963    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
17964        self.resolver.resolve_source_table(name)
17965    }
17966}
17967
17968// The Engine is a Resolver because it implements the constituent traits.
17969impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
17970
17971// The Engine provides functions by delegating to its internal resolver.
17972impl<R> crate::traits::FunctionProvider for Engine<R>
17973where
17974    R: EvaluationContext,
17975{
17976    fn planning_semantic_revision(&self) -> Option<u64> {
17977        self.resolver.planning_semantic_revision()
17978    }
17979
17980    fn get_function(
17981        &self,
17982        prefix: &str,
17983        name: &str,
17984    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
17985        self.resolver.get_function(prefix, name)
17986    }
17987
17988    fn get_function_for_planning(
17989        &self,
17990        prefix: &str,
17991        name: &str,
17992    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
17993        self.resolver.get_function_for_planning(prefix, name)
17994    }
17995}
17996
17997impl<R> Engine<R>
17998where
17999    R: EvaluationContext,
18000{
18001    /// Semantic used coordinates exclude graph-only dependency placeholders.
18002    ///
18003    /// Non-empty base/overlay/computed cells come from Arrow storage, while
18004    /// scalar and array formulas come from graph formula kinds even before
18005    /// their results are materialized. The legacy graph fallback is omitted:
18006    /// `load_packed_to_vertex` entries are either represented by those sources
18007    /// or are `Empty` dependency placeholders, not a third value authority.
18008    pub(crate) fn semantic_used_rows_for_columns(
18009        &self,
18010        sheet: &str,
18011        start_col: u32,
18012        end_col: u32,
18013    ) -> Option<(u32, u32)> {
18014        let arrow_bounds = self
18015            .sheet_store()
18016            .sheet(sheet)
18017            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18018        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18019        Self::union_used_bounds(arrow_bounds, formula_bounds)
18020    }
18021
18022    pub(crate) fn semantic_used_cols_for_rows(
18023        &self,
18024        sheet: &str,
18025        start_row: u32,
18026        end_row: u32,
18027    ) -> Option<(u32, u32)> {
18028        let arrow_bounds = self
18029            .sheet_store()
18030            .sheet(sheet)
18031            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18032        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18033        Self::union_used_bounds(arrow_bounds, formula_bounds)
18034    }
18035}
18036
18037// Override EvaluationContext to provide thread pool access
18038impl<R> crate::traits::EvaluationContext for Engine<R>
18039where
18040    R: EvaluationContext,
18041{
18042    fn clock(&self) -> &dyn crate::timezone::ClockProvider {
18043        &self.clock
18044    }
18045
18046    fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
18047        self.thread_pool.as_ref()
18048    }
18049
18050    fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
18051        self.active_cancel_flag.clone()
18052    }
18053
18054    fn chunk_hint(&self) -> Option<usize> {
18055        // Use a simple heuristic from configuration (stripe width * height) as a default hint.
18056        let hint =
18057            (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
18058        Some(hint.clamp(1024, 1 << 20)) // clamp between 1K and ~1M
18059    }
18060
18061    fn volatile_level(&self) -> crate::traits::VolatileLevel {
18062        self.config.volatile_level
18063    }
18064
18065    fn workbook_seed(&self) -> u64 {
18066        self.config.workbook_seed
18067    }
18068
18069    fn recalc_epoch(&self) -> u64 {
18070        self.recalc_epoch
18071    }
18072
18073    fn workbook_sheet_count(&self) -> Option<usize> {
18074        Some(self.graph.sheet_reg().active_len())
18075    }
18076
18077    fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
18078        self.graph.sheet_reg().active_position(sheet)
18079    }
18080
18081    fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
18082        self.sheet_index_by_name(current_sheet)
18083    }
18084
18085    fn inspect_reference(
18086        &self,
18087        reference: &ReferenceType,
18088        current_sheet: &str,
18089    ) -> Result<Option<ReferenceInfo>, ExcelError> {
18090        let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
18091            let sheet_id = self
18092                .graph
18093                .sheet_id(sheet_name)
18094                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18095            let sheet_index = self
18096                .graph
18097                .sheet_reg()
18098                .active_position_by_id(sheet_id)
18099                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18100            Ok((sheet_id, sheet_index))
18101        };
18102
18103        let cell_info =
18104            |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
18105                let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18106                let row0 = row
18107                    .checked_sub(1)
18108                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18109                let col0 = col
18110                    .checked_sub(1)
18111                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18112                Ok(ReferenceInfo {
18113                    first_sheet_index: Some(sheet_index),
18114                    sheet_count: Some(1),
18115                    first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18116                })
18117            };
18118
18119        let range_info = |sheet_name: &str,
18120                          start_row: Option<u32>,
18121                          start_col: Option<u32>|
18122         -> Result<ReferenceInfo, ExcelError> {
18123            let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18124            let row = start_row.unwrap_or(1);
18125            let col = start_col.unwrap_or(1);
18126            let row0 = row
18127                .checked_sub(1)
18128                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18129            let col0 = col
18130                .checked_sub(1)
18131                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18132            Ok(ReferenceInfo {
18133                first_sheet_index: Some(sheet_index),
18134                sheet_count: Some(1),
18135                first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18136            })
18137        };
18138
18139        let info = match reference {
18140            ReferenceType::Cell {
18141                sheet, row, col, ..
18142            } => {
18143                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18144                cell_info(sheet_name, *row, *col)?
18145            }
18146            ReferenceType::Range {
18147                sheet,
18148                start_row,
18149                start_col,
18150                ..
18151            } => {
18152                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18153                range_info(sheet_name, *start_row, *start_col)?
18154            }
18155            ReferenceType::Cell3D {
18156                sheet_first,
18157                sheet_last,
18158                row,
18159                col,
18160                ..
18161            } => {
18162                let first = cell_info(sheet_first, *row, *col)?;
18163                ReferenceInfo {
18164                    first_sheet_index: first.first_sheet_index,
18165                    sheet_count: self
18166                        .graph
18167                        .sheet_reg()
18168                        .active_span_len(sheet_first, sheet_last),
18169                    first_cell: first.first_cell,
18170                }
18171            }
18172            ReferenceType::Range3D {
18173                sheet_first,
18174                sheet_last,
18175                start_row,
18176                start_col,
18177                ..
18178            } => {
18179                let first = range_info(sheet_first, *start_row, *start_col)?;
18180                ReferenceInfo {
18181                    first_sheet_index: first.first_sheet_index,
18182                    sheet_count: self
18183                        .graph
18184                        .sheet_reg()
18185                        .active_span_len(sheet_first, sheet_last),
18186                    first_cell: first.first_cell,
18187                }
18188            }
18189            ReferenceType::NamedRange(name) => {
18190                let current_id = self
18191                    .graph
18192                    .sheet_id(current_sheet)
18193                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18194                let named = self
18195                    .graph
18196                    .resolve_name_entry(name, current_id)
18197                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18198                match &named.definition {
18199                    NamedDefinition::Cell(cell) => ReferenceInfo {
18200                        first_sheet_index: self
18201                            .graph
18202                            .sheet_reg()
18203                            .active_position_by_id(cell.sheet_id),
18204                        sheet_count: Some(1),
18205                        first_cell: Some(*cell),
18206                    },
18207                    NamedDefinition::Range(range) => ReferenceInfo {
18208                        first_sheet_index: self
18209                            .graph
18210                            .sheet_reg()
18211                            .active_position_by_id(range.start.sheet_id),
18212                        sheet_count: Some(1),
18213                        first_cell: Some(range.start),
18214                    },
18215                    NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
18216                        ReferenceInfo {
18217                            first_sheet_index: None,
18218                            sheet_count: None,
18219                            first_cell: None,
18220                        }
18221                    }
18222                }
18223            }
18224            ReferenceType::Table(tref) => {
18225                let table = self
18226                    .graph
18227                    .resolve_table_entry(&tref.name)
18228                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18229                ReferenceInfo {
18230                    first_sheet_index: self
18231                        .graph
18232                        .sheet_reg()
18233                        .active_position_by_id(table.range.start.sheet_id),
18234                    sheet_count: Some(1),
18235                    first_cell: Some(table.range.start),
18236                }
18237            }
18238            ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18239        };
18240
18241        Ok(Some(info))
18242    }
18243
18244    fn resolve_spill_reference(
18245        &self,
18246        anchor: &ReferenceType,
18247        current_sheet: &str,
18248    ) -> Result<ReferenceType, ExcelError> {
18249        let no_spill = || {
18250            ExcelError::new(ExcelErrorKind::Ref)
18251                .with_message("Spill reference anchor has no current spill")
18252        };
18253        // The rectangle keeps the operand's sheet qualification: an
18254        // unqualified anchor on the current sheet yields an unqualified range.
18255        let mut qualify = true;
18256        let anchor_cell = match anchor {
18257            ReferenceType::Cell {
18258                sheet, row, col, ..
18259            } => {
18260                qualify = sheet.is_some();
18261                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18262                let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(no_spill)?;
18263                if *row == 0 || *col == 0 {
18264                    return Err(no_spill());
18265                }
18266                CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true))
18267            }
18268            ReferenceType::NamedRange(name) => {
18269                let current_id = self.graph.sheet_id(current_sheet).ok_or_else(no_spill)?;
18270                let named = self
18271                    .graph
18272                    .resolve_name_entry(name, current_id)
18273                    .ok_or_else(no_spill)?;
18274                match &named.definition {
18275                    NamedDefinition::Cell(cell) => *cell,
18276                    NamedDefinition::Range(range)
18277                        if range.start.sheet_id == range.end.sheet_id
18278                            && range.start.coord.row() == range.end.coord.row()
18279                            && range.start.coord.col() == range.end.coord.col() =>
18280                    {
18281                        range.start
18282                    }
18283                    _ => {
18284                        return Err(ExcelError::new(ExcelErrorKind::Ref)
18285                            .with_message("Spill reference name must refer to a single cell"));
18286                    }
18287                }
18288            }
18289            _ => {
18290                return Err(ExcelError::new(ExcelErrorKind::Ref)
18291                    .with_message("Spill reference operand must be a single cell"));
18292            }
18293        };
18294        let vertex = self
18295            .graph
18296            .get_vertex_id_for_address(&anchor_cell)
18297            .ok_or_else(no_spill)?;
18298        // Policy: the spill operator applies only to dynamic arrays, never
18299        // to a legacy fixed-extent formula (including a single-cell one).
18300        if self.graph.fixed_single_arrays.contains(&vertex)
18301            || self.graph.fixed_array_shapes.contains_key(&vertex)
18302        {
18303            return Err(ExcelError::new(ExcelErrorKind::Ref)
18304                .with_message("Spill reference to a fixed array formula"));
18305        }
18306        let (first, last) = match self.graph.spill_extent_for_anchor(vertex) {
18307            Some(extent) => extent,
18308            // FORM211: a declared anchor (source spill identity) holding a
18309            // current scalar result is a 1x1 spill. A malformed registry
18310            // entry stays `#REF!`, as do blocked/error/empty results.
18311            None if !self.graph.spill_registry_has_anchor(vertex)
18312                && self.graph.is_current_declared_dynamic_anchor(vertex)
18313                && self.declared_anchor_holds_scalar(anchor_cell) =>
18314            {
18315                (anchor_cell, anchor_cell)
18316            }
18317            None => return Err(no_spill()),
18318        };
18319        let sheet_name = qualify.then(|| self.graph.sheet_name(first.sheet_id).to_string());
18320        let (sr, sc) = (first.coord.row() + 1, first.coord.col() + 1);
18321        let (er, ec) = (last.coord.row() + 1, last.coord.col() + 1);
18322        // The rectangle is concrete, so its bounds are absolute.
18323        Ok(if sr == er && sc == ec {
18324            ReferenceType::cell_with_abs(sheet_name, sr, sc, true, true)
18325        } else {
18326            ReferenceType::Range {
18327                sheet: sheet_name,
18328                start_row: Some(sr),
18329                start_col: Some(sc),
18330                end_row: Some(er),
18331                end_col: Some(ec),
18332                start_row_abs: true,
18333                start_col_abs: true,
18334                end_row_abs: true,
18335                end_col_abs: true,
18336            }
18337        })
18338    }
18339
18340    fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
18341        let sheet_name = self.graph.sheet_name(cell.sheet_id);
18342        if sheet_name.is_empty() {
18343            return Err(ExcelError::new(ExcelErrorKind::Ref));
18344        }
18345        let row = cell.coord.row() + 1;
18346        let col = cell.coord.col() + 1;
18347
18348        if let Some(entries) = self.staged_formulas.get(sheet_name)
18349            && let Some(text) = entries.get(row, col)
18350        {
18351            return Ok(Some(if text.starts_with('=') {
18352                text.to_owned()
18353            } else {
18354                format!("={text}")
18355            }));
18356        }
18357
18358        let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
18359            return Ok(None);
18360        };
18361        Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
18362    }
18363
18364    fn used_rows_for_columns(
18365        &self,
18366        sheet: &str,
18367        start_col: u32,
18368        end_col: u32,
18369    ) -> Option<(u32, u32)> {
18370        // Union Arrow-backed used-region with formula rows that have not been materialized yet.
18371        let sheet_id = self.graph.sheet_id(sheet)?;
18372        let snap = self.data_snapshot_id();
18373        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18374            guard
18375                .as_ref()
18376                .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
18377        }) {
18378            return cached;
18379        }
18380
18381        let arrow_bounds = self
18382            .sheet_store()
18383            .sheet(sheet)
18384            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18385        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18386        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18387            Some(bounds)
18388        } else {
18389            let sc0 = start_col.saturating_sub(1);
18390            let ec0 = end_col.saturating_sub(1);
18391            self.graph
18392                .used_row_bounds_for_columns(sheet_id, sc0, ec0)
18393                .map(|(a0, b0)| (a0 + 1, b0 + 1))
18394        };
18395
18396        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18397            guard
18398                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18399                .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
18400        }
18401
18402        computed
18403    }
18404
18405    fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
18406        // Union Arrow-backed used-region with formula columns that have not been materialized yet.
18407        let sheet_id = self.graph.sheet_id(sheet)?;
18408        let snap = self.data_snapshot_id();
18409        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18410            guard
18411                .as_ref()
18412                .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
18413        }) {
18414            return cached;
18415        }
18416
18417        let arrow_bounds = self
18418            .sheet_store()
18419            .sheet(sheet)
18420            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18421        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18422        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18423            Some(bounds)
18424        } else {
18425            let sr0 = start_row.saturating_sub(1);
18426            let er0 = end_row.saturating_sub(1);
18427            self.graph
18428                .used_col_bounds_for_rows(sheet_id, sr0, er0)
18429                .map(|(a0, b0)| (a0 + 1, b0 + 1))
18430        };
18431
18432        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18433            guard
18434                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18435                .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
18436        }
18437
18438        computed
18439    }
18440
18441    fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
18442        let _ = self.graph.sheet_id(sheet)?;
18443        // Excel-like upper bounds; we expose something finite but large.
18444        // Backends may override with real bounds.
18445        Some((1_048_576, 16_384)) // 1048576 rows, 16384 cols (XFD)
18446    }
18447
18448    fn data_snapshot_id(&self) -> u64 {
18449        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
18450    }
18451
18452    fn backend_caps(&self) -> crate::traits::BackendCaps {
18453        crate::traits::BackendCaps {
18454            streaming: true,
18455            used_region: true,
18456            write: false,
18457            tables: false,
18458            async_stream: false,
18459        }
18460    }
18461
18462    fn build_lookup_index(
18463        &self,
18464        view: &RangeView<'_>,
18465        axis: LookupAxis,
18466    ) -> Option<Arc<LookupIndex>> {
18467        self.build_lookup_index_impl(view, axis)
18468    }
18469
18470    // Flats removed
18471
18472    fn date_system(&self) -> crate::engine::DateSystem {
18473        self.config.date_system
18474    }
18475    /// New: resolve a reference into a RangeView (Phase 2 API)
18476    fn resolve_range_view<'c>(
18477        &'c self,
18478        reference: &ReferenceType,
18479        current_sheet: &str,
18480    ) -> Result<RangeView<'c>, ExcelError> {
18481        match reference {
18482            ReferenceType::External(ext) => {
18483                let name = ext.raw.as_str();
18484                match ext.kind {
18485                    formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
18486                        let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
18487                            return Err(ExcelError::new(ExcelErrorKind::Name)
18488                                .with_message(format!("Undefined name: {name}")));
18489                        };
18490                        let version = source
18491                            .version
18492                            .or_else(|| self.resolver.source_scalar_version(name));
18493                        let v = self.resolve_source_scalar_cached(name, version)?;
18494                        Ok(RangeView::from_owned_rows(
18495                            vec![vec![v]],
18496                            self.config.date_system,
18497                        ))
18498                    }
18499                    formualizer_parse::parser::ExternalRefKind::Range { .. } => {
18500                        let Some(source) = self.graph.resolve_source_table_entry(name) else {
18501                            // A deferred (whole-row/column) external range
18502                            // evaluates to #REF!, as it did before the parser
18503                            // kept it whole (see `unbound_external_range_defers`).
18504                            let kind =
18505                                if crate::engine::refs::unbound_external_range_defers(&ext.kind) {
18506                                    ExcelErrorKind::Ref
18507                                } else {
18508                                    ExcelErrorKind::Name
18509                                };
18510                            return Err(ExcelError::new(kind)
18511                                .with_message(format!("Undefined table: {name}")));
18512                        };
18513                        let version = source
18514                            .version
18515                            .or_else(|| self.resolver.source_table_version(name));
18516                        let table = self.resolve_source_table_cached(name, version)?;
18517                        let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
18518                        self.source_table_to_range_view(table.as_ref(), &spec)
18519                    }
18520                }
18521            }
18522            ReferenceType::Range { .. } => {
18523                let shared = self.resolve_shared_ref(reference, current_sheet)?;
18524                let formualizer_common::SheetRef::Range(range) = shared else {
18525                    return Err(ExcelError::new(ExcelErrorKind::Ref));
18526                };
18527                // No context sheet is available here, so an unresolved locator
18528                // is #REF! rather than a guess (issue #110).
18529                let sheet_id = match range.sheet {
18530                    formualizer_common::SheetLocator::Id(id) => id,
18531                    formualizer_common::SheetLocator::Current
18532                    | formualizer_common::SheetLocator::Name(_) => {
18533                        return Err(ExcelError::new(ExcelErrorKind::Ref));
18534                    }
18535                };
18536                let sheet_name = self.graph.sheet_name(sheet_id);
18537
18538                let bounded_range = if range.start_row.is_some()
18539                    && range.start_col.is_some()
18540                    && range.end_row.is_some()
18541                    && range.end_col.is_some()
18542                {
18543                    Some(RangeRef::try_from_shared(range.as_ref())?)
18544                } else {
18545                    None
18546                };
18547
18548                let sr = bounded_range
18549                    .as_ref()
18550                    .map(|r| r.start.coord.row() + 1)
18551                    .or_else(|| range.start_row.map(|b| b.index + 1));
18552                let sc = bounded_range
18553                    .as_ref()
18554                    .map(|r| r.start.coord.col() + 1)
18555                    .or_else(|| range.start_col.map(|b| b.index + 1));
18556                let er = bounded_range
18557                    .as_ref()
18558                    .map(|r| r.end.coord.row() + 1)
18559                    .or_else(|| range.end_row.map(|b| b.index + 1));
18560                let ec = bounded_range
18561                    .as_ref()
18562                    .map(|r| r.end.coord.col() + 1)
18563                    .or_else(|| range.end_col.map(|b| b.index + 1));
18564
18565                let extent = resolve_used_extent_with_fallback(
18566                    OpenRangeBounds {
18567                        start_row: sr,
18568                        start_column: sc,
18569                        end_row: er,
18570                        end_column: ec,
18571                    },
18572                    ExtentPolicy::EvaluationCompat {
18573                        fallback_row: None,
18574                        fallback_column: None,
18575                    },
18576                    || {
18577                        self.sheet_bounds(sheet_name)
18578                            .map(|_| self.config.max_open_ended_rows)
18579                    },
18580                    || {
18581                        self.sheet_bounds(sheet_name)
18582                            .map(|_| self.config.max_open_ended_cols)
18583                    },
18584                    |first, last| self.used_rows_for_columns(sheet_name, first, last),
18585                    |first, last| self.used_cols_for_rows(sheet_name, first, last),
18586                );
18587                let (sr, sc, er, ec) = extent
18588                    .map(|extent| {
18589                        (
18590                            extent.start_row,
18591                            extent.start_column,
18592                            extent.end_row,
18593                            extent.end_column,
18594                        )
18595                    })
18596                    .unwrap_or((1, 1, 0, 0));
18597
18598                if self.force_materialize_range_views {
18599                    if er < sr || ec < sc {
18600                        return Ok(RangeView::from_owned_rows(
18601                            Vec::new(),
18602                            self.config.date_system,
18603                        ));
18604                    }
18605                    let h = (er - sr + 1) as u64;
18606                    let w = (ec - sc + 1) as u64;
18607                    let cell_count = h.saturating_mul(w);
18608                    if cell_count <= self.config.spill.max_spill_cells as u64 {
18609                        let mut rows: Vec<Vec<LiteralValue>> = Vec::with_capacity(h as usize);
18610                        for r in sr..=er {
18611                            let mut rowv: Vec<LiteralValue> = Vec::with_capacity(w as usize);
18612                            for c in sc..=ec {
18613                                rowv.push(
18614                                    self.get_cell_value(sheet_name, r, c)
18615                                        .unwrap_or(LiteralValue::Empty),
18616                                );
18617                            }
18618                            rows.push(rowv);
18619                        }
18620                        return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
18621                    }
18622                }
18623
18624                let Some(asheet) = self.sheet_store().sheet(sheet_name) else {
18625                    return Ok(RangeView::from_owned_rows(
18626                        Vec::new(),
18627                        self.config.date_system,
18628                    ));
18629                };
18630
18631                let rv = if er < sr || ec < sc {
18632                    asheet.range_view(1, 1, 0, 0)
18633                } else {
18634                    let sr0 = sr.saturating_sub(1) as usize;
18635                    let sc0 = sc.saturating_sub(1) as usize;
18636                    let er0 = er.saturating_sub(1) as usize;
18637                    let ec0 = ec.saturating_sub(1) as usize;
18638                    asheet.range_view(sr0, sc0, er0, ec0)
18639                };
18640
18641                Ok(rv)
18642            }
18643            ReferenceType::Cell { .. } => {
18644                let shared = self.resolve_shared_ref(reference, current_sheet)?;
18645                let formualizer_common::SheetRef::Cell(cell) = shared else {
18646                    return Err(ExcelError::new(ExcelErrorKind::Ref));
18647                };
18648                let addr = CellRef::try_from_shared(cell)?;
18649                let sheet_id = addr.sheet_id;
18650                let sheet_name = self.graph.sheet_name(sheet_id);
18651                let row = addr.coord.row() + 1;
18652                let col = addr.coord.col() + 1;
18653
18654                if self.force_materialize_range_views {
18655                    let v = self
18656                        .get_cell_value(sheet_name, row, col)
18657                        .unwrap_or(LiteralValue::Empty);
18658                    return Ok(RangeView::from_owned_rows(
18659                        vec![vec![v]],
18660                        self.config.date_system,
18661                    ));
18662                }
18663
18664                if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
18665                    let r0 = row.saturating_sub(1) as usize;
18666                    let c0 = col.saturating_sub(1) as usize;
18667                    let rv = asheet.range_view(r0, c0, r0, c0);
18668                    Ok(rv)
18669                } else {
18670                    let v = self
18671                        .get_cell_value(sheet_name, row, col)
18672                        .unwrap_or(LiteralValue::Empty);
18673                    Ok(RangeView::from_owned_rows(
18674                        vec![vec![v]],
18675                        self.config.date_system,
18676                    ))
18677                }
18678            }
18679            ReferenceType::NamedRange(name) => {
18680                if let Some(current_id) = self.graph.sheet_id(current_sheet)
18681                    && let Some(named) = self.graph.resolve_name_entry(name, current_id)
18682                {
18683                    match &named.definition {
18684                        NamedDefinition::Cell(cell_ref) => {
18685                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
18686                            if self.force_materialize_range_views {
18687                                let v = self
18688                                    .get_cell_value(
18689                                        sheet_name,
18690                                        cell_ref.coord.row() + 1,
18691                                        cell_ref.coord.col() + 1,
18692                                    )
18693                                    .unwrap_or(LiteralValue::Empty);
18694                                return Ok(RangeView::from_owned_rows(
18695                                    vec![vec![v]],
18696                                    self.config.date_system,
18697                                ));
18698                            } else {
18699                                let asheet = self
18700                                    .sheet_store()
18701                                    .sheet(sheet_name)
18702                                    .expect("Arrow sheet missing for named cell");
18703                                let r0 = cell_ref.coord.row() as usize;
18704                                let c0 = cell_ref.coord.col() as usize;
18705                                let rv = asheet.range_view(r0, c0, r0, c0);
18706                                return Ok(rv);
18707                            }
18708                        }
18709                        NamedDefinition::Range(range_ref) => {
18710                            let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
18711                            let sr = range_ref.start.coord.row() + 1;
18712                            let sc = range_ref.start.coord.col() + 1;
18713                            let er = range_ref.end.coord.row() + 1;
18714                            let ec = range_ref.end.coord.col() + 1;
18715                            if self.force_materialize_range_views {
18716                                let h = (er.saturating_sub(sr) + 1) as u64;
18717                                let w = (ec.saturating_sub(sc) + 1) as u64;
18718                                let cell_count = h.saturating_mul(w);
18719                                if cell_count <= self.config.spill.max_spill_cells as u64 {
18720                                    let mut rows: Vec<Vec<LiteralValue>> =
18721                                        Vec::with_capacity(h as usize);
18722                                    for r in sr..=er {
18723                                        let mut rowv: Vec<LiteralValue> =
18724                                            Vec::with_capacity(w as usize);
18725                                        for c in sc..=ec {
18726                                            rowv.push(
18727                                                self.get_cell_value(sheet_name, r, c)
18728                                                    .unwrap_or(LiteralValue::Empty),
18729                                            );
18730                                        }
18731                                        rows.push(rowv);
18732                                    }
18733                                    return Ok(RangeView::from_owned_rows(
18734                                        rows,
18735                                        self.config.date_system,
18736                                    ));
18737                                }
18738                            }
18739                            let asheet = self
18740                                .sheet_store()
18741                                .sheet(sheet_name)
18742                                .expect("Arrow sheet missing for named range");
18743                            let sr0 = range_ref.start.coord.row() as usize;
18744                            let sc0 = range_ref.start.coord.col() as usize;
18745                            let er0 = range_ref.end.coord.row() as usize;
18746                            let ec0 = range_ref.end.coord.col() as usize;
18747                            let rv = asheet.range_view(sr0, sc0, er0, ec0);
18748                            return Ok(rv);
18749                        }
18750                        NamedDefinition::Literal(v) => {
18751                            return Ok(RangeView::from_owned_rows(
18752                                vec![vec![v.clone()]],
18753                                self.config.date_system,
18754                            ));
18755                        }
18756                        NamedDefinition::Formula { .. } => {
18757                            if let Some(value) = self.graph.get_value(named.vertex) {
18758                                return Ok(RangeView::from_owned_rows(
18759                                    vec![vec![value]],
18760                                    self.config.date_system,
18761                                ));
18762                            }
18763                        }
18764                    }
18765                }
18766
18767                if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
18768                    let version = source
18769                        .version
18770                        .or_else(|| self.resolver.source_scalar_version(name));
18771                    let v = self.resolve_source_scalar_cached(name, version)?;
18772                    return Ok(RangeView::from_owned_rows(
18773                        vec![vec![v]],
18774                        self.config.date_system,
18775                    ));
18776                }
18777
18778                let data = self.resolver.resolve_named_range_reference(name)?;
18779                Ok(RangeView::from_owned_rows(data, self.config.date_system))
18780            }
18781            ReferenceType::Table(tref) => {
18782                if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
18783                    let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
18784                    let asheet = self
18785                        .sheet_store()
18786                        .sheet(sheet_name)
18787                        .expect("Arrow sheet missing for table reference");
18788
18789                    let sr0 = table.range.start.coord.row() as usize;
18790                    let sc0 = table.range.start.coord.col() as usize;
18791                    let er0 = table.range.end.coord.row() as usize;
18792                    let ec0 = table.range.end.coord.col() as usize;
18793
18794                    let has_totals = table.totals_row;
18795                    let has_headers = table.header_row;
18796                    let data_sr = if has_headers {
18797                        sr0.saturating_add(1)
18798                    } else {
18799                        sr0
18800                    };
18801                    let data_er = if has_totals {
18802                        er0.saturating_sub(1)
18803                    } else {
18804                        er0
18805                    };
18806
18807                    let select = |sr: usize, sc: usize, er: usize, ec: usize| {
18808                        if sr > er || sc > ec {
18809                            asheet.range_view(1, 1, 0, 0)
18810                        } else {
18811                            asheet.range_view(sr, sc, er, ec)
18812                        }
18813                    };
18814
18815                    let av = match &tref.specifier {
18816                        None => {
18817                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18818                                "Table reference without specifier is unsupported".to_string(),
18819                            ));
18820                        }
18821                        Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
18822                            let Some(idx) = table.col_index(col) else {
18823                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18824                                    "Column refers to unknown table column".to_string(),
18825                                ));
18826                            };
18827                            let c0 = sc0 + idx;
18828                            select(data_sr, c0, data_er, c0)
18829                        }
18830                        Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
18831                            start,
18832                            end,
18833                        )) => {
18834                            let Some(si) = table.col_index(start) else {
18835                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18836                                    "Column range refers to unknown column(s)".to_string(),
18837                                ));
18838                            };
18839                            let Some(ei) = table.col_index(end) else {
18840                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18841                                    "Column range refers to unknown column(s)".to_string(),
18842                                ));
18843                            };
18844                            let (mut a, mut b) = (si, ei);
18845                            if a > b {
18846                                std::mem::swap(&mut a, &mut b);
18847                            }
18848                            let c_start = sc0 + a;
18849                            let c_end = sc0 + b;
18850                            select(data_sr, c_start, data_er, c_end)
18851                        }
18852                        Some(formualizer_parse::parser::TableSpecifier::All)
18853                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18854                            formualizer_parse::parser::SpecialItem::All,
18855                        )) => select(sr0, sc0, er0, ec0),
18856                        Some(formualizer_parse::parser::TableSpecifier::Data)
18857                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18858                            formualizer_parse::parser::SpecialItem::Data,
18859                        )) => select(data_sr, sc0, data_er, ec0),
18860                        Some(formualizer_parse::parser::TableSpecifier::Headers)
18861                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18862                            formualizer_parse::parser::SpecialItem::Headers,
18863                        )) => {
18864                            if !has_headers {
18865                                asheet.range_view(1, 1, 0, 0)
18866                            } else {
18867                                select(sr0, sc0, sr0, ec0)
18868                            }
18869                        }
18870                        Some(formualizer_parse::parser::TableSpecifier::Totals)
18871                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18872                            formualizer_parse::parser::SpecialItem::Totals,
18873                        )) => {
18874                            if !has_totals {
18875                                asheet.range_view(1, 1, 0, 0)
18876                            } else {
18877                                select(er0, sc0, er0, ec0)
18878                            }
18879                        }
18880                        Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18881                            formualizer_parse::parser::SpecialItem::ThisRow,
18882                        )) => {
18883                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18884                                "@ (This Row) requires table-aware context; not yet supported"
18885                                    .to_string(),
18886                            ));
18887                        }
18888                        Some(formualizer_parse::parser::TableSpecifier::Row(_))
18889                        | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
18890                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18891                                "Complex structured references not yet supported".to_string(),
18892                            ));
18893                        }
18894                    };
18895
18896                    return Ok(av);
18897                }
18898
18899                if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
18900                    let version = source
18901                        .version
18902                        .or_else(|| self.resolver.source_table_version(&tref.name));
18903                    let table = self.resolve_source_table_cached(&tref.name, version)?;
18904                    return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
18905                }
18906
18907                // Fallback: materialize via Resolver::resolve_range_like tranche 1.
18908                // A table nobody defines (an unbound reference kept by the
18909                // BestEffort preparation policy) is `#NAME?`, the kind Strict
18910                // reports at preparation, not the resolver's "not implemented".
18911                let boxed = self
18912                    .resolve_range_like(&ReferenceType::Table(tref.clone()))
18913                    .map_err(|e| {
18914                        if e.kind == ExcelErrorKind::NImpl {
18915                            ExcelError::new(ExcelErrorKind::Name)
18916                                .with_message(format!("Unknown table: {}", tref.name))
18917                        } else {
18918                            e
18919                        }
18920                    })?;
18921                let owned = boxed.materialise().into_owned();
18922                Ok(RangeView::from_owned_rows(owned, self.config.date_system))
18923            }
18924            ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
18925                Err(ExcelError::new(ExcelErrorKind::NImpl)
18926                    .with_message("3D references are not yet supported".to_string()))
18927            }
18928        }
18929    }
18930
18931    fn resolve_cell_format(
18932        &self,
18933        sheet: Option<&str>,
18934        row: u32,
18935        col: u32,
18936        current_sheet: &str,
18937    ) -> Option<crate::format::FormatId> {
18938        self.effective_format_id(sheet.unwrap_or(current_sheet), row, col)
18939    }
18940
18941    fn format_class(
18942        &self,
18943        format: crate::format::FormatId,
18944    ) -> Option<formualizer_common::numfmt::FormatClass> {
18945        self.format_registry.class(format).cloned()
18946    }
18947
18948    fn record_cell_derived_format(
18949        &self,
18950        sheet: &str,
18951        row: u32,
18952        col: u32,
18953        format: Option<crate::format::FormatId>,
18954    ) {
18955        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
18956            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18957            self.record_derived_format_at(cell, format);
18958        }
18959    }
18960
18961    fn resolve_cell_reference_value(
18962        &self,
18963        sheet: Option<&str>,
18964        row: u32,
18965        col: u32,
18966        current_sheet: &str,
18967    ) -> Result<LiteralValue, ExcelError> {
18968        let sheet_name = sheet.unwrap_or(current_sheet);
18969        if self.graph.sheet_id(sheet_name).is_none() {
18970            return Err(ExcelError::new(ExcelErrorKind::Ref));
18971        }
18972        Ok(self
18973            .get_cell_value(sheet_name, row, col)
18974            .unwrap_or(LiteralValue::Empty))
18975    }
18976
18977    fn resolve_cell_reference_value_formatted(
18978        &self,
18979        sheet: Option<&str>,
18980        row: u32,
18981        col: u32,
18982        current_sheet: &str,
18983    ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
18984        // `resolve_cell_reference_value` + `resolve_cell_format` with one
18985        // sheet lookup of each kind.
18986        let sheet_name = sheet.unwrap_or(current_sheet);
18987        let Some(sheet_id) = self.graph.sheet_id(sheet_name) else {
18988            return Err(ExcelError::new(ExcelErrorKind::Ref));
18989        };
18990        let asheet = self.arrow_sheets.sheet(sheet_name);
18991        Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
18992    }
18993
18994    fn build_criteria_mask(
18995        &self,
18996        view: &RangeView<'_>,
18997        col_in_view: usize,
18998        pred: &crate::args::CriteriaPredicate,
18999    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19000        #[cfg(any(test, feature = "test-support"))]
19001        criteria_mask_test_hooks::note_mask(view.dims().0);
19002        if view.dims().1 == 0 {
19003            return None;
19004        }
19005        // If the view is logically open-ended but the backing sheet has no physical rows,
19006        // treat the mask as empty (0-len) rather than attempting to build a huge mask.
19007        let sheet_rows = view.sheet().nrows as usize;
19008        if sheet_rows == 0 || view.start_row() >= sheet_rows {
19009            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
19010        }
19011        compute_criteria_mask(view, col_in_view, pred)
19012    }
19013
19014    fn build_row_visibility_mask(
19015        &self,
19016        view: &RangeView<'_>,
19017        mode: VisibilityMaskMode,
19018    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19019        self.build_row_visibility_mask_for_view(view, mode)
19020    }
19021
19022    fn nested_subtotal_cells(
19023        &self,
19024        view: &RangeView<'_>,
19025        include_aggregate: bool,
19026    ) -> Option<Vec<(usize, usize, usize)>> {
19027        self.nested_subtotal_cells_for_view(view, include_aggregate)
19028    }
19029}
19030
19031impl<R> Engine<R>
19032where
19033    R: EvaluationContext,
19034{
19035    fn clear_spill_projection_and_mirror(
19036        &mut self,
19037        anchor_vertex: VertexId,
19038        delta: Option<&mut DeltaCollector>,
19039    ) {
19040        let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
19041        if spill_cells.is_empty() {
19042            return;
19043        }
19044
19045        if let Some(delta) = delta
19046            && delta.mode != DeltaMode::Off
19047        {
19048            let empty = LiteralValue::Empty;
19049            for cell in spill_cells.iter() {
19050                let sheet_name = self.graph.sheet_name(cell.sheet_id);
19051                let old = self
19052                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19053                    .unwrap_or(LiteralValue::Empty);
19054                if old != empty {
19055                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19056                }
19057            }
19058        }
19059
19060        self.graph.clear_spill_region(anchor_vertex);
19061        if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
19062            self.record_structural_change(scope);
19063        }
19064
19065        if self.config.arrow_storage_enabled
19066            && self.config.delta_overlay_enabled
19067            && self.config.write_formula_overlay_enabled
19068        {
19069            let empty = LiteralValue::Empty;
19070            for cell in spill_cells.iter() {
19071                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19072                self.mirror_value_to_computed_overlay(
19073                    &sheet_name,
19074                    cell.coord.row() + 1,
19075                    cell.coord.col() + 1,
19076                    &empty,
19077                );
19078            }
19079        }
19080    }
19081
19082    /// Apply the evaluation outcome for one cyclic SCC: stamp `#CIRC!` on its
19083    /// (optionally filtered) members via `stamp_cycle_error`.
19084    ///
19085    /// This is the single per-SCC application point used by every schedule
19086    /// consumer walking `Schedule::units` (pre-work for #112, where cyclic
19087    /// SCCs will gain runtime verdicts instead of an unconditional stamp).
19088    ///
19089    /// `dirty_filter` preserves the recalc-plan quirk: when `Some(dirty)`,
19090    /// only members present in the set are stamped.
19091    ///
19092    /// Returns the number of vertices stamped (0 when a filter excludes every
19093    /// member), so callers can keep their site-specific `cycle_errors`
19094    /// accounting.
19095    fn apply_cycle_outcome(
19096        &mut self,
19097        cycle: &[VertexId],
19098        mut delta: Option<&mut DeltaCollector>,
19099        dirty_filter: Option<&FxHashSet<VertexId>>,
19100    ) -> usize {
19101        let circ_error = LiteralValue::Error(
19102            ExcelError::new(ExcelErrorKind::Circ)
19103                .with_message("Circular dependency detected".to_string()),
19104        );
19105        let mut stamped = 0usize;
19106        for &vertex_id in cycle {
19107            if let Some(filter) = dirty_filter
19108                && !filter.contains(&vertex_id)
19109            {
19110                continue;
19111            }
19112            self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
19113            stamped += 1;
19114        }
19115        stamped
19116    }
19117
19118    /// Stamp a vertex with `#CIRC!` as part of cycle handling.
19119    ///
19120    /// Unlike a bare `update_vertex_value`, this first tears down any spill the
19121    /// vertex previously anchored: it clears the spilled cells, releases the graph
19122    /// spill registry, drops any lingering region reservation, and mirrors the
19123    /// cleared cells into the computed overlay — the same teardown a normal scalar/
19124    /// error result performs (see `apply_non_array_result_from_parallel` /
19125    /// `clear_spill_projection_and_mirror`). Without this, a #CIRC stamp on a former
19126    /// spill anchor would leave stale spilled values and a reserved region behind
19127    /// (issue #111).
19128    ///
19129    /// When `delta` is provided, the cleared spill cells are recorded (by
19130    /// `clear_spill_projection_and_mirror`) and the anchor's own #CIRC change is
19131    /// recorded here, matching how other result paths emit deltas.
19132    fn stamp_cycle_error(
19133        &mut self,
19134        vertex_id: VertexId,
19135        circ_error: &LiteralValue,
19136        mut delta: Option<&mut DeltaCollector>,
19137    ) {
19138        // Tear down any previous spill projection/region before overwriting the anchor.
19139        if self.graph.spill_registry_has_anchor(vertex_id) {
19140            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
19141        }
19142        // Drop any reservation that was never committed (defensive; normally released
19143        // on the prior successful commit).
19144        self.spill_mgr.release_owner(vertex_id);
19145
19146        // Record the anchor's own #CIRC delta, like other result paths.
19147        if let Some(d) = delta
19148            && d.mode != DeltaMode::Off
19149            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
19150        {
19151            let sheet_name = self.graph.sheet_name(cell.sheet_id);
19152            let old = self
19153                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19154                .unwrap_or(LiteralValue::Empty);
19155            if old != *circ_error {
19156                d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19157            }
19158        }
19159
19160        self.graph.update_vertex_value_ref(vertex_id, circ_error);
19161        self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
19162    }
19163
19164    /// Dispatch point for one `ScheduleUnit::Cycle` (RFC #112, Stage 2).
19165    ///
19166    /// * `CycleDetection::Static` — today's behavior, byte-for-byte: stamp
19167    ///   `#CIRC!` on the (optionally dirty-filtered) members.
19168    /// * `CycleDetection::Runtime` — evaluate the SCC via
19169    ///   [`Self::evaluate_scc_unit`]. The recalc-plan dirty quirk maps to:
19170    ///   no dirty member → skip the task entirely (values stand); any dirty
19171    ///   member → the whole SCC evaluates (an SCC cannot be partially
19172    ///   evaluated).
19173    ///
19174    /// Returns the number of `#CIRC!`-stamped vertices, so call sites can
19175    /// keep their `cycle_errors` accounting (`> 0` ⇒ count the unit).
19176    fn handle_cycle_unit(
19177        &mut self,
19178        cycle: &[VertexId],
19179        mut delta: Option<&mut DeltaCollector>,
19180        dirty_filter: Option<&FxHashSet<VertexId>>,
19181        cancel_flag: Option<&AtomicBool>,
19182    ) -> Result<usize, ExcelError> {
19183        self.resource_checkpoint(cycle.len() as u64)?;
19184        match self.config.cycle.detection {
19185            CycleDetection::Static => {
19186                Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
19187            }
19188            CycleDetection::Runtime => {
19189                if let Some(filter) = dirty_filter
19190                    && !cycle.iter().any(|v| filter.contains(v))
19191                {
19192                    return Ok(0);
19193                }
19194                // Both policies share `evaluate_scc_unit`; they differ only
19195                // in the settle loop's live-cycle arm (Error stamps,
19196                // Iterate keeps passing — RFC #113).
19197                self.evaluate_scc_unit(cycle, delta, cancel_flag)
19198            }
19199        }
19200    }
19201
19202    /// Evaluate one statically-cyclic SCC under `CycleDetection::Runtime`
19203    /// (design doc `formualizer-stage2-scc-evaluation-design.md` §3; contract
19204    /// spec §3; Iterate policy arm per RFC #113).
19205    ///
19206    /// Phantom SCCs (live-acyclic) produce ordinary values under both
19207    /// policies; live cycles get `#CIRC!` with live-cycle-only blast radius
19208    /// under `CyclePolicy::Error`, or Excel-style iterative calculation
19209    /// (converge per spec §6 or cap at `max_iterations` passes) under
19210    /// `CyclePolicy::Iterate`. Runs sequentially on the
19211    /// coordinating thread; commits are write-through per member (no
19212    /// `ComputedWriteBuffer` — that buffer is scoped to layer evaluation and
19213    /// always flushed before a Cycle unit runs, G1), so later members' scalar
19214    /// *and* range reads observe earlier members' results through the overlay
19215    /// cascade. Deltas are recorded once per member at end of task (G11).
19216    ///
19217    /// Returns the number of vertices stamped `#CIRC!`.
19218    ///
19219    /// `pub(crate)` so tests can drive SCC shapes (e.g. name-vertex members)
19220    /// that ingest-time cycle rejection makes unreachable via public edits.
19221    pub(crate) fn evaluate_scc_unit(
19222        &mut self,
19223        cycle: &[VertexId],
19224        mut delta: Option<&mut DeltaCollector>,
19225        cancel_flag: Option<&AtomicBool>,
19226    ) -> Result<usize, ExcelError> {
19227        struct SccMember {
19228            vertex: VertexId,
19229            cell: Option<CellRef>,
19230        }
19231
19232        let task_start = crate::instant::FzInstant::now();
19233
19234        // ── 0. Member order (spec §7.13): cells ascending (sheet, row, col);
19235        // name vertices after, lexicographic by folded canonical name; any
19236        // other vertex kind (defensive — `get_evaluation_vertices` only emits
19237        // formula/name kinds) last by id, never evaluated.
19238        let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
19239        let mut name_members: Vec<(VertexId, String)> = Vec::new();
19240        let mut other_members: Vec<VertexId> = Vec::new();
19241        for &v in cycle {
19242            match self.graph.get_vertex_kind(v) {
19243                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
19244                    match self.graph.get_cell_ref(v) {
19245                        Some(cell) => cell_members.push((v, cell)),
19246                        None => other_members.push(v),
19247                    }
19248                }
19249                VertexKind::NamedScalar | VertexKind::NamedArray => {
19250                    match self.graph.name_key_for_vertex(v) {
19251                        Some(key) => name_members.push((v, key)),
19252                        None => other_members.push(v),
19253                    }
19254                }
19255                _ => other_members.push(v),
19256            }
19257        }
19258        cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
19259        name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
19260        other_members.sort_unstable();
19261
19262        let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
19263        let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
19264        let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
19265        for (v, c) in &cell_members {
19266            members.push(SccMember {
19267                vertex: *v,
19268                cell: Some(*c),
19269            });
19270        }
19271        for (v, _) in &name_members {
19272            members.push(SccMember {
19273                vertex: *v,
19274                cell: None,
19275            });
19276        }
19277        for v in &other_members {
19278            members.push(SccMember {
19279                vertex: *v,
19280                cell: None,
19281            });
19282        }
19283        let n = members.len();
19284        // Indices addressable by the collector (cells + names); `other`
19285        // members can be neither edge sources nor targets.
19286        let recordable = cell_refs.len() + name_keys.len();
19287
19288        let circ_error = LiteralValue::Error(
19289            ExcelError::new(ExcelErrorKind::Circ)
19290                .with_message("Circular dependency detected".to_string()),
19291        );
19292
19293        // ── 0b. Spec-§4 persistence repair: structural edits clear computed
19294        // overlays wholesale (`clear_computed_overlay_after_row/_col`), but
19295        // an iterating member's committed value is cycle STATE, not a
19296        // recomputable cache — and in canonical mode the overlay is its ONLY
19297        // home. If the overlay entry vanished since the last recalc, re-seed
19298        // it from the end-of-recalc snapshot (`iterative_state_values`) so
19299        // pass-1 reads (scalar AND range, via the overlay cascade) observe
19300        // the persisted value instead of silently restarting at Empty→0.
19301        // (Found by the iterate edge corpus: inserting/deleting an unrelated
19302        // row reset accumulators, violating spec §4/§7.15.)
19303        if !self.iterative_state_values.is_empty() {
19304            let restore: Vec<(VertexId, LiteralValue)> = members
19305                .iter()
19306                .filter_map(|m| {
19307                    let cell = m.cell?;
19308                    let persisted = self.iterative_state_values.get(&m.vertex)?;
19309                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
19310                    let overlay = self
19311                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19312                        .unwrap_or(LiteralValue::Empty);
19313                    if matches!(overlay, LiteralValue::Empty) {
19314                        Some((m.vertex, persisted.clone()))
19315                    } else {
19316                        None
19317                    }
19318                })
19319                .collect();
19320            for (vertex, value) in restore {
19321                self.mirror_vertex_value_to_overlay(vertex, &value);
19322            }
19323        }
19324
19325        // ── 1. Pre-task value snapshot (overlay-first for cells — G3; the
19326        // graph value map may be evicted in value-cache-disabled mode).
19327        let snapshot: Vec<LiteralValue> = members
19328            .iter()
19329            .map(|m| match m.cell {
19330                Some(cell) => {
19331                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
19332                    self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19333                        .unwrap_or(LiteralValue::Empty)
19334                }
19335                None => self
19336                    .graph
19337                    .get_value(m.vertex)
19338                    .unwrap_or(LiteralValue::Empty),
19339            })
19340            .collect();
19341
19342        // ── 2. Pre-scan: spill anchors (FormulaArray) are stamped `#CIRC!`
19343        // with full spill teardown (spec §7.9, #115) and excluded from
19344        // evaluation. They stay recordable edge TARGETS (readers see `#CIRC!`
19345        // and propagate). Non-evaluable defensive members are excluded too.
19346        let mut excluded = vec![false; n];
19347        let mut last_value = snapshot.clone();
19348        let mut stamped = 0usize;
19349        for (i, m) in members.iter().enumerate() {
19350            match self.graph.get_vertex_kind(m.vertex) {
19351                VertexKind::FormulaArray => {
19352                    // Deltas for the cleared spill-region cells (non-members)
19353                    // can only be recorded here; the anchor's own delta is
19354                    // covered by the end-of-task snapshot comparison (dedup).
19355                    self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
19356                    excluded[i] = true;
19357                    last_value[i] = circ_error.clone();
19358                    stamped += 1;
19359                }
19360                VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
19361                _ => excluded[i] = true,
19362            }
19363        }
19364
19365        let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
19366
19367        // Per-member live out-edges, refreshed whenever a member re-runs.
19368        let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
19369        // Position of each member in the most recent pass (-1 = did not run).
19370        let mut pos: Vec<i64> = vec![-1; n];
19371        // Whether each member's committed value changed in the most recent pass.
19372        let mut changed = vec![false; n];
19373
19374        // Evaluate-and-commit one member; returns Ok(true) when the member was
19375        // stamped `#CIRC!` (array result — would-be spill anchor, spec §7.9).
19376        macro_rules! run_member {
19377            ($i:expr) => {{
19378                let i: usize = $i;
19379                let m = &members[i];
19380                if i < recordable {
19381                    collector.set_current(i as u32);
19382                }
19383                let value = {
19384                    let ctx = RecordingContext::new(&*self, &collector);
19385                    match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
19386                        Ok(v) => v,
19387                        Err(e) => LiteralValue::Error(e),
19388                    }
19389                };
19390                let is_cell_formula = m.cell.is_some();
19391                if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
19392                    // A member that *would* spill inside an SCC gets the
19393                    // conservative §7.9 verdict. It has never spilled before
19394                    // (a prior spill would make it FormulaArray, pre-stamped
19395                    // above), so there is no projection to tear down.
19396                    self.stamp_cycle_error(m.vertex, &circ_error, None);
19397                    excluded[i] = true;
19398                    stamped += 1;
19399                    changed[i] = last_value[i] != circ_error;
19400                    last_value[i] = circ_error.clone();
19401                } else {
19402                    self.graph.update_vertex_value_ref(m.vertex, &value);
19403                    self.mirror_vertex_value_to_overlay(m.vertex, &value);
19404                    // §7.14 invariant (G2): a formula member must never be
19405                    // shadowed by a user/delta overlay entry, or iteration
19406                    // reads would silently diverge from committed values.
19407                    #[cfg(debug_assertions)]
19408                    if let Some(cell) = m.cell {
19409                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19410                        debug_assert!(
19411                            self.read_delta_overlay_cell(
19412                                &sheet_name,
19413                                cell.coord.row() + 1,
19414                                cell.coord.col() + 1
19415                            )
19416                            .is_none(),
19417                            "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
19418                            cell.coord.row() + 1,
19419                            cell.coord.col() + 1
19420                        );
19421                    }
19422                    changed[i] = last_value[i] != value;
19423                    last_value[i] = value;
19424                }
19425            }};
19426        }
19427
19428        let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
19429            if let Some(flag) = flag
19430                && flag.load(Ordering::Relaxed)
19431            {
19432                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
19433                    .with_message("Evaluation cancelled during SCC evaluation".to_string()));
19434            }
19435            Ok(())
19436        };
19437
19438        // ── 3. Pass 1: all evaluable members in member order.
19439        check_cancel(cancel_flag)?;
19440        let mut passes = 1usize;
19441        {
19442            let mut p = 0i64;
19443            for i in 0..n {
19444                if excluded[i] {
19445                    continue;
19446                }
19447                run_member!(i);
19448                pos[i] = p;
19449                p += 1;
19450            }
19451        }
19452
19453        // ── 4. Settle loop (design doc §3 step 4; RFC #113 policy arm).
19454        //
19455        // Acyclic classifications settle stale readers exactly (identical
19456        // under both policies — phantom SCCs never iterate). A witnessed
19457        // live cycle dispatches on policy: `Error` stamps `#CIRC!` and
19458        // stops; `Iterate` keeps running full passes over all members in
19459        // member order until converged (spec §6) or capped at
19460        // `max_iterations` total passes. A live cycle that only appears
19461        // mid-settle takes the same arm, and a cycle that dissolves
19462        // mid-iteration falls back to exact acyclic settling.
19463        //
19464        // Defensive acyclic budget: the acyclic settle is monotone, so more
19465        // than |SCC| + 2 settle passes can only be a bug; cap hits stamp the
19466        // remainder and set telemetry. Tracked via `settle_passes` so
19467        // iteration passes (legitimately many) don't consume the budget.
19468        let policy = self.config.cycle.policy;
19469        let cap = n + 2;
19470        let mut witnessed_cycles = 0usize;
19471        let mut capped = false;
19472        // ── Iterate-policy state ──
19473        let mut iterating = false;
19474        let mut converged = false;
19475        let mut exact_fixed_point = false;
19476        // Values committed by the last *full* pass; `None` until the first
19477        // iteration pass runs (pass 1 has no predecessor to compare against)
19478        // and reset when a settle pass runs (no cross-kind comparisons).
19479        let mut prev_pass: Option<Vec<LiteralValue>> = None;
19480        // Final-round convergence stats (overwritten per round so the values
19481        // reported are the ones observed at stop).
19482        let mut iter_max_delta = 0f64;
19483        let mut iter_nan_converged = 0usize;
19484        // Acyclic stale-reader re-eval passes (defensive budget; under pure
19485        // Error flow `1 + settle_passes == passes`, preserving Stage-2
19486        // behavior exactly).
19487        let mut settle_passes = 0usize;
19488        loop {
19489            // Drain this pass's recordings; members that ran replace their
19490            // out-edge set, members that didn't keep last-known edges.
19491            let drained = collector.take_edges();
19492            for i in 0..n {
19493                if pos[i] >= 0 {
19494                    out_edges[i].clear();
19495                }
19496            }
19497            for (from, to) in drained {
19498                debug_assert!(
19499                    pos[from as usize] >= 0,
19500                    "edge from a member that did not run"
19501                );
19502                out_edges[from as usize].push(to);
19503            }
19504            let mut edges: Vec<(u32, u32)> = Vec::new();
19505            for (i, outs) in out_edges.iter().enumerate() {
19506                if excluded[i] {
19507                    continue;
19508                }
19509                for &t in outs {
19510                    edges.push((i as u32, t));
19511                }
19512            }
19513            edges.sort_unstable();
19514            edges.dedup();
19515
19516            let analysis = analyze_live_graph(n, &edges);
19517
19518            if analysis.cycle_count > 0 {
19519                // Classification repeats every iteration pass under
19520                // `Iterate`; record the widest single witness instead of
19521                // accumulating so the count stays "distinct live cycles".
19522                witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
19523                match policy {
19524                    CyclePolicy::Error => {
19525                        // POLICY (Error): stamp every member of a live cycle,
19526                        // then one settling pass over the remaining members in
19527                        // live-topological order so error propagation
19528                        // downstream is consistent (spec §3.4). Blast radius =
19529                        // live cycles only.
19530                        for i in 0..n {
19531                            if analysis.in_cycle[i] && !excluded[i] {
19532                                self.stamp_cycle_error(members[i].vertex, &circ_error, None);
19533                                excluded[i] = true;
19534                                last_value[i] = circ_error.clone();
19535                                stamped += 1;
19536                            }
19537                        }
19538                        check_cancel(cancel_flag)?;
19539                        let order: Vec<usize> = analysis
19540                            .topo
19541                            .iter()
19542                            .map(|&i| i as usize)
19543                            .filter(|&i| !excluded[i])
19544                            .collect();
19545                        if !order.is_empty() {
19546                            passes += 1;
19547                            for i in order {
19548                                run_member!(i);
19549                            }
19550                        }
19551                        break;
19552                    }
19553                    CyclePolicy::Iterate {
19554                        max_iterations,
19555                        max_change,
19556                    } => {
19557                        // POLICY (Iterate), spec §3.5/§6.
19558                        iterating = true;
19559
19560                        // Convergence test: the full pass that just completed
19561                        // vs the previous full pass, per the spec-§6 rules.
19562                        // `prev_pass` is `None` until an iteration pass has
19563                        // run — pass 1 has no predecessor, so no convergence
19564                        // test occurs before the second pass (spec §6).
19565                        if let Some(prev) = &prev_pass {
19566                            let mut round_max_delta = 0f64;
19567                            let mut round_nan = 0usize;
19568                            let mut all_converged = true;
19569                            let mut round_exact = true;
19570                            for i in 0..n {
19571                                if excluded[i] {
19572                                    // Stamped mid-iteration (array result,
19573                                    // §7.9): the value is pinned and cannot
19574                                    // change again — trivially settled.
19575                                    continue;
19576                                }
19577                                let out = crate::engine::convergence::values_converged(
19578                                    &prev[i],
19579                                    &last_value[i],
19580                                    max_change,
19581                                    self.config.date_system,
19582                                );
19583                                if out.nan_converged {
19584                                    round_nan += 1;
19585                                }
19586                                if let Some(d) = out.abs_delta {
19587                                    round_max_delta = round_max_delta.max(d);
19588                                    if d != 0.0 {
19589                                        round_exact = false;
19590                                    }
19591                                }
19592                                if !out.converged {
19593                                    all_converged = false;
19594                                }
19595                            }
19596                            // Overwrite (not max): telemetry reports the
19597                            // round observed at stop.
19598                            iter_max_delta = round_max_delta;
19599                            iter_nan_converged = round_nan;
19600                            if all_converged {
19601                                exact_fixed_point = round_exact;
19602                                converged = true;
19603                                break;
19604                            }
19605                        }
19606
19607                        // ── Pass-counting reconciliation (spec §6/§7.6):
19608                        // `max_iterations` counts TOTAL passes, pass 1
19609                        // included, and pass 1 has already run by the time a
19610                        // live cycle is first witnessed here. The budget is
19611                        // therefore checked BEFORE evaluating anything more:
19612                        // with `max_iterations: 1` we stop right here — each
19613                        // member was evaluated exactly once this recalc (the
19614                        // Excel accumulator contract) and no convergence test
19615                        // ran (`prev_pass` is still `None`). Capping keeps
19616                        // the last committed values and is NOT an error
19617                        // (Excel parity); telemetry records it.
19618                        if passes >= max_iterations as usize {
19619                            capped = true;
19620                            break;
19621                        }
19622
19623                        check_cancel(cancel_flag)?;
19624                        // One more full pass over every evaluable member in
19625                        // member order (Gauss–Seidel: each commit is visible
19626                        // to later members within the pass). Live edges
19627                        // re-record — guards can flip near convergence
19628                        // (§7.3) — so classification repeats next time
19629                        // around, and a cycle that dissolves drops back to
19630                        // the exact acyclic settle below.
19631                        prev_pass = Some(last_value.clone());
19632                        for x in pos.iter_mut() {
19633                            *x = -1;
19634                        }
19635                        changed.fill(false);
19636                        passes += 1;
19637                        let mut p = 0i64;
19638                        for i in 0..n {
19639                            if excluded[i] {
19640                                continue;
19641                            }
19642                            run_member!(i);
19643                            pos[i] = p;
19644                            p += 1;
19645                        }
19646                        continue;
19647                    }
19648                }
19649            }
19650
19651            // Acyclic: find stale readers — members whose live read of `to`
19652            // happened before `to`'s value changed in the pass that just ran.
19653            let mut stale: Vec<usize> = Vec::new();
19654            for i in 0..n {
19655                if excluded[i] {
19656                    continue;
19657                }
19658                let is_stale = out_edges[i].iter().any(|&t| {
19659                    let t = t as usize;
19660                    changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
19661                });
19662                if is_stale {
19663                    stale.push(i);
19664                }
19665            }
19666            if stale.is_empty() {
19667                break; // values exact — phantom SCC (or dissolved live cycle)
19668            }
19669            if 1 + settle_passes >= cap {
19670                // Defensive only; hitting this is a bug (loud telemetry).
19671                capped = true;
19672                for (i, m) in members.iter().enumerate() {
19673                    if !excluded[i] {
19674                        self.stamp_cycle_error(m.vertex, &circ_error, None);
19675                        excluded[i] = true;
19676                        last_value[i] = circ_error.clone();
19677                        stamped += 1;
19678                    }
19679                }
19680                break;
19681            }
19682
19683            check_cancel(cancel_flag)?;
19684            // Re-evaluate stale readers in live-topo order, recording fresh
19685            // edges (branches may flip on re-eval — spec §7.3 — which is why
19686            // classification repeats).
19687            // A settle pass is a partial sweep: drop the full-pass baseline
19688            // so a live cycle (re)appearing afterwards never compares values
19689            // across mixed pass kinds.
19690            prev_pass = None;
19691            let topo_pos = analysis.topo_positions();
19692            stale.sort_unstable_by_key(|&i| topo_pos[i]);
19693            for x in pos.iter_mut() {
19694                *x = -1;
19695            }
19696            changed.fill(false);
19697            passes += 1;
19698            settle_passes += 1;
19699            for (p, i) in stale.into_iter().enumerate() {
19700                run_member!(i);
19701                pos[i] = p as i64;
19702            }
19703        }
19704
19705        // Post-work boundary: members are committed write-through, so a
19706        // member that ran across a live cancellation is already visible; the
19707        // task still must not complete (no delta, retention or iteration
19708        // state). The members stay dirty for the retry, as with the
19709        // mid-task checks above.
19710        self.live_cancellation_after_work(SCC_CANCELLED)?;
19711
19712        // Iteration that ended because the live cycle dissolved and the
19713        // acyclic settle reached exactness counts as converged (values are
19714        // exact, strictly better than threshold-converged). The defensive
19715        // settle cap (`capped` + stamping) is not.
19716        if iterating && !converged && !capped {
19717            converged = true;
19718        }
19719
19720        // ── 5. End of task: one delta per member whose final value differs
19721        // from the pre-task snapshot (spec §3 side-effect rule, G11).
19722        collector.clear_current();
19723        if let Some(d) = delta
19724            && d.mode != DeltaMode::Off
19725        {
19726            for (i, m) in members.iter().enumerate() {
19727                if let Some(cell) = m.cell
19728                    && last_value[i] != snapshot[i]
19729                {
19730                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19731                }
19732            }
19733        }
19734
19735        // Members of an SCC that iterated re-evaluate on EVERY recalc, like
19736        // Excel's circular cells: register them for the end-of-recalc
19737        // volatile-like redirty (see `pending_iterative_redirty`). Marking
19738        // any one member propagates around the (strongly connected) SCC and
19739        // to downstream dependents, but all members are registered so the
19740        // contract survives partial structural edits between recalcs.
19741        //
19742        // Exception (#368): an SCC that stopped on an
19743        // exact fixed point — every member reproduced its previous value
19744        // bit-for-bit before the pass cap, no NaN identity, no volatile or
19745        // dynamic-reference member — cannot change on a re-run with the same
19746        // inputs, so it is retained clean. The dirty graph decides when it
19747        // runs again. Tolerance-only convergence (|Δ| < max_change but ≠ 0),
19748        // capped SCCs (including the `max_iterations: 1` accumulator
19749        // contract) and volatile cycles keep the per-recalc redirty.
19750        // Whatever the outcome, this task supersedes any earlier retention
19751        // of its members.
19752        if !self.retained_scc_members.is_empty() {
19753            for m in members.iter() {
19754                self.retained_scc_members.remove(&m.vertex);
19755            }
19756        }
19757        if iterating {
19758            let retain = converged
19759                && !capped
19760                && exact_fixed_point
19761                && iter_nan_converged == 0
19762                && members
19763                    .iter()
19764                    .all(|m| !self.graph.is_volatile(m.vertex) && !self.graph.is_dynamic(m.vertex));
19765            if retain {
19766                if self.retained_scc_members.is_empty() {
19767                    self.retained_scc_config_fingerprint = self.retained_scc_config_fingerprint();
19768                    self.retained_scc_function_epoch_seen =
19769                        crate::function_registry::semantic_epoch();
19770                    self.retained_scc_provider_revision_seen =
19771                        self.resolver.planning_semantic_revision();
19772                }
19773                let scc_id = self.next_retained_scc_id;
19774                self.next_retained_scc_id = self.next_retained_scc_id.wrapping_add(1);
19775                for (i, m) in members.iter().enumerate() {
19776                    self.retained_scc_members.insert(m.vertex, scc_id);
19777                    // §4 persistence snapshot, written once: retained
19778                    // members do not pass through `redirty_for_next_recalc`.
19779                    if matches!(last_value[i], LiteralValue::Empty) {
19780                        self.iterative_state_values.remove(&m.vertex);
19781                    } else {
19782                        self.iterative_state_values
19783                            .insert(m.vertex, last_value[i].clone());
19784                    }
19785                }
19786            } else {
19787                self.pending_iterative_redirty
19788                    .extend(members.iter().map(|m| m.vertex));
19789            }
19790        } else if !self.iterative_state_values.is_empty() {
19791            // The cycle dissolved (phantom settle or `#CIRC!` stamping):
19792            // these members are ordinary formulas again and must not carry
19793            // stale iteration state into a future cycle.
19794            for m in members.iter() {
19795                self.iterative_state_values.remove(&m.vertex);
19796            }
19797        }
19798
19799        {
19800            let t = &mut self.last_cycle_telemetry;
19801            t.static_sccs += 1;
19802            if witnessed_cycles == 0 && stamped == 0 && !capped {
19803                t.phantom_sccs += 1;
19804            }
19805            t.live_cycles_witnessed += witnessed_cycles;
19806            t.circ_cells_stamped += stamped;
19807            t.settle_passes_total += passes;
19808            t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
19809            if iterating {
19810                t.iterated_sccs += 1;
19811                if converged {
19812                    t.converged_sccs += 1;
19813                }
19814                t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
19815                t.nan_converged += iter_nan_converged;
19816            }
19817            if capped {
19818                t.capped_sccs += 1;
19819            }
19820            t.elapsed_ms += task_start.elapsed().as_millis();
19821        }
19822
19823        Ok(stamped)
19824    }
19825
19826    /// Recorded sibling of [`Self::evaluate_vertex_immutable`]: evaluates one
19827    /// SCC member's AST via an [`Interpreter`] over a [`RecordingContext`] so
19828    /// reads that actually occur are captured as live edges. Value semantics
19829    /// must match `evaluate_vertex_immutable` exactly (including the missing-
19830    /// AST `Number(0.0)` quirk, G14); named Cell/Range/Literal definitions
19831    /// delegate to it after recording the definition region by hand (those
19832    /// reads bypass the context).
19833    fn evaluate_vertex_recorded(
19834        &self,
19835        vertex_id: VertexId,
19836        ctx: &RecordingContext<'_, R>,
19837        collector: &LiveEdgeCollector,
19838    ) -> Result<LiteralValue, ExcelError> {
19839        if !self.graph.vertex_exists(vertex_id) {
19840            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
19841                .with_message(format!("Vertex not found: {vertex_id:?}")));
19842        }
19843
19844        let kind = self.graph.get_vertex_kind(vertex_id);
19845        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
19846
19847        match kind {
19848            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
19849                let Some(view) = self.graph.formula_view(vertex_id) else {
19850                    return Ok(LiteralValue::Number(0.0)); // G14 quirk
19851                };
19852                let sheet_name = self.graph.sheet_name(sheet_id);
19853                let cell_ref = self
19854                    .graph
19855                    .get_cell_ref(vertex_id)
19856                    .expect("cell ref for vertex");
19857                let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
19858                interpreter
19859                    .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
19860                    .map(|cv| {
19861                        let format = cv.format_id();
19862                        self.record_derived_format(vertex_id, format);
19863                        self.materialize_formula_result(vertex_id, cv)
19864                    })
19865                    .or_else(|error| self.fit_formula_error(vertex_id, error))
19866            }
19867            VertexKind::NamedScalar | VertexKind::NamedArray => {
19868                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
19869                    ExcelError::new(ExcelErrorKind::Name)
19870                        .with_message("Named range metadata missing".to_string())
19871                })?;
19872
19873                match &named_range.definition {
19874                    NamedDefinition::Formula { ast, .. } => {
19875                        let context_sheet = match named_range.scope {
19876                            NameScope::Sheet(id) => id,
19877                            NameScope::Workbook => sheet_id,
19878                        };
19879                        let sheet_name = self.graph.sheet_name(context_sheet);
19880                        let cell_ref = self
19881                            .graph
19882                            .get_cell_ref(vertex_id)
19883                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
19884                        let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
19885                        if kind == VertexKind::NamedScalar {
19886                            interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
19887                        } else {
19888                            match interpreter.evaluate_ast(ast) {
19889                                Ok(cv) => match cv.into_literal() {
19890                                    v @ LiteralValue::Array(_) => Ok(v),
19891                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
19892                                },
19893                                Err(err) => Ok(LiteralValue::Error(err)),
19894                            }
19895                        }
19896                    }
19897                    NamedDefinition::Cell(cell_ref) => {
19898                        // The definition is read via direct grid access in
19899                        // `evaluate_vertex_immutable`; record the live edge
19900                        // by hand before delegating.
19901                        collector.record_scalar(
19902                            cell_ref.sheet_id,
19903                            cell_ref.coord.row(),
19904                            cell_ref.coord.col(),
19905                        );
19906                        self.evaluate_vertex_immutable(vertex_id)
19907                    }
19908                    NamedDefinition::Range(range_ref) => {
19909                        if range_ref.start.sheet_id == range_ref.end.sheet_id {
19910                            collector.record_rect(
19911                                range_ref.start.sheet_id,
19912                                range_ref.start.coord.row(),
19913                                range_ref.start.coord.col(),
19914                                range_ref.end.coord.row(),
19915                                range_ref.end.coord.col(),
19916                            );
19917                        }
19918                        self.evaluate_vertex_immutable(vertex_id)
19919                    }
19920                    NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
19921                }
19922            }
19923            _ => self.evaluate_vertex_immutable(vertex_id),
19924        }
19925    }
19926
19927    /// Pending source occupancy is independent of formula preparation and value caches.
19928    fn pending_spill_occupied(&self, anchor: CellRef, end_row: u32, end_col: u32) -> bool {
19929        let sheet = self.graph.sheet_name(anchor.sheet_id);
19930        let package = self
19931            .staged_formulas
19932            .get(sheet)
19933            .and_then(|staged| staged.deferred_package.as_ref());
19934        self.staged_formula_index.occupies_spill(
19935            sheet,
19936            (anchor.coord.row() + 1, anchor.coord.col() + 1),
19937            (end_row + 1, end_col + 1),
19938            |point| package.is_some_and(|package| package.suppressed.contains(&point)),
19939        )
19940    }
19941
19942    fn remember_pending_spill(
19943        &mut self,
19944        vertex: VertexId,
19945        anchor: CellRef,
19946        region: Region,
19947    ) -> Result<(), ExcelError> {
19948        self.cancellation_checkpoint("pending spill occupancy")?;
19949        self.resource_checkpoint(1)?;
19950        if let Some(entry) = self
19951            .blocked_pending_spills
19952            .iter_mut()
19953            .find(|entry| entry.0 == vertex)
19954        {
19955            *entry = (vertex, anchor, region);
19956            return Ok(());
19957        }
19958        if self.blocked_pending_spills.len() == self.blocked_pending_spills.capacity() {
19959            // Geometric growth avoids copying every existing retry entry for
19960            // every new blocked anchor. Admit the entire capacity increment.
19961            let additional = self.blocked_pending_spills.capacity().max(1);
19962            let bytes =
19963                (additional as u64)
19964                    .saturating_mul(std::mem::size_of::<(VertexId, CellRef, Region)>() as u64);
19965            if let Some(ledger) = self.active_resource_ledger.as_mut() {
19966                ledger
19967                    .reserve_retained(bytes)
19968                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19969                self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
19970            }
19971            if self
19972                .blocked_pending_spills
19973                .try_reserve_exact(additional)
19974                .is_err()
19975            {
19976                if let Some(ledger) = self.active_resource_ledger.as_mut() {
19977                    ledger
19978                        .release_retained(bytes)
19979                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19980                    self.source_cache_accounted -= bytes;
19981                }
19982                return Err(crate::engine::ResourceLedgerError::Exhausted(
19983                    formualizer_common::ResourceExhaustionDetail {
19984                        reason: formualizer_common::ResourceExhaustionReason::RetainedMemory,
19985                        limit: u64::MAX,
19986                        observed: bytes,
19987                        request_id: None,
19988                    },
19989                )
19990                .into_excel_error());
19991            }
19992        }
19993        self.blocked_pending_spills.push((vertex, anchor, region));
19994        Ok(())
19995    }
19996
19997    // Successful edits wake only intersecting attempted regions. Keep the entry
19998    // until evaluation: logged edits may still roll back, and materializing a
19999    // pending formula must not destroy its anchor's occupancy retry information.
20000    fn invalidate_pending_spills(&mut self, scope: StructuralScope) {
20001        if let StructuralScope::RemovedSheet(sheet) = scope {
20002            self.blocked_pending_spills
20003                .retain(|entry| entry.1.sheet_id != sheet);
20004            return;
20005        }
20006        for &(vertex, anchor, region) in &self.blocked_pending_spills {
20007            let affected = match scope {
20008                StructuralScope::Cell { sheet, row, col } => {
20009                    region.intersects(&Region::point(sheet, row, col))
20010                }
20011                StructuralScope::Region(changed) => region.intersects(&changed),
20012                StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => {
20013                    region.sheet_id() == sheet
20014                }
20015                StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => true,
20016            };
20017            if affected
20018                && self.graph.vertex_exists(vertex)
20019                && self.graph.get_cell_ref(vertex) == Some(anchor)
20020                && matches!(
20021                    self.graph.get_vertex_kind(vertex),
20022                    VertexKind::FormulaScalar | VertexKind::FormulaArray
20023                )
20024            {
20025                self.graph.mark_vertex_dirty(vertex);
20026            }
20027        }
20028    }
20029
20030    fn guard_pending_spill_commit(
20031        &mut self,
20032        anchor_vertex: VertexId,
20033        targets: &[CellRef],
20034    ) -> Result<(), ExcelError> {
20035        let Some(anchor) = self.graph.get_cell_ref(anchor_vertex) else {
20036            return Ok(());
20037        };
20038        let Some(last) = targets.last() else {
20039            return Ok(());
20040        };
20041        let occupied = self.pending_spill_occupied(anchor, last.coord.row(), last.coord.col())
20042            || self
20043                .graph
20044                .table_intersects_spill(anchor, last.coord.row(), last.coord.col());
20045        if (occupied
20046            || self
20047                .blocked_pending_spills
20048                .iter()
20049                .any(|entry| entry.0 == anchor_vertex))
20050            && let Err(error) = self.remember_pending_spill(
20051                anchor_vertex,
20052                anchor,
20053                Region::rect(
20054                    anchor.sheet_id,
20055                    anchor.coord.row(),
20056                    last.coord.row(),
20057                    anchor.coord.col(),
20058                    last.coord.col(),
20059                ),
20060            )
20061        {
20062            self.spill_mgr.release_owner(anchor_vertex);
20063            return Err(error);
20064        }
20065        if occupied {
20066            self.spill_mgr.release_owner(anchor_vertex);
20067            return Err(ExcelError::new(ExcelErrorKind::Spill)
20068                .with_message("Spill blocked")
20069                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20070                    expected_rows: last.coord.row() - anchor.coord.row() + 1,
20071                    expected_cols: last.coord.col() - anchor.coord.col() + 1,
20072                }));
20073        }
20074        Ok(())
20075    }
20076
20077    /// Commit spill via shim and mirror resulting cells into Arrow overlay.
20078    fn commit_spill_and_mirror(
20079        &mut self,
20080        anchor_vertex: VertexId,
20081        targets: &[CellRef],
20082        rows: Vec<Vec<LiteralValue>>,
20083        delta: Option<&mut DeltaCollector>,
20084    ) -> Result<(), ExcelError> {
20085        self.guard_pending_spill_commit(anchor_vertex, targets)?;
20086        let prev_spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
20087
20088        if let Some(delta) = delta
20089            && delta.mode != DeltaMode::Off
20090        {
20091            let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20092                targets.iter().copied().collect();
20093            let empty = LiteralValue::Empty;
20094
20095            // Clears (prev - targets)
20096            for cell in prev_spill_cells.iter() {
20097                if target_set.contains(cell) {
20098                    continue;
20099                }
20100                let sheet_name = self.graph.sheet_name(cell.sheet_id);
20101                let old = self
20102                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20103                    .unwrap_or(LiteralValue::Empty);
20104                if old != empty {
20105                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20106                }
20107            }
20108
20109            // Writes (targets)
20110            if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
20111                let width = rows[0].len();
20112                for (idx, cell) in targets.iter().enumerate() {
20113                    let r_off = idx / width;
20114                    let c_off = idx % width;
20115                    let new = rows
20116                        .get(r_off)
20117                        .and_then(|r| r.get(c_off))
20118                        .cloned()
20119                        .unwrap_or(LiteralValue::Empty);
20120                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
20121                    let old = self
20122                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20123                        .unwrap_or(LiteralValue::Empty);
20124                    if old != new {
20125                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20126                    }
20127                }
20128            } else {
20129                // Degenerate shapes: if we have targets but no rows, treat as writing Empty.
20130                for cell in targets.iter() {
20131                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
20132                    let old = self
20133                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20134                        .unwrap_or(LiteralValue::Empty);
20135                    if !matches!(old, LiteralValue::Empty) {
20136                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20137                    }
20138                }
20139            }
20140        }
20141
20142        // Commit via shim (releases locks). When the graph value cache is disabled (Arrow-canonical
20143        // values), plan/commit must consult Arrow storage to detect non-empty value blockers.
20144        let arrow_sheets = &self.arrow_sheets;
20145        self.spill_mgr.commit_array_with_value_probe(
20146            &mut self.graph,
20147            anchor_vertex,
20148            targets,
20149            rows.clone(),
20150            |g, cell| {
20151                let sheet_name = g.sheet_name(cell.sheet_id);
20152                let asheet = arrow_sheets.sheet(sheet_name)?;
20153                let r0 = cell.coord.row() as usize;
20154                let c0 = cell.coord.col() as usize;
20155                let v = asheet.get_cell_value(r0, c0);
20156                if matches!(v, LiteralValue::Empty) {
20157                    None
20158                } else {
20159                    Some(v)
20160                }
20161            },
20162        )?;
20163
20164        self.blocked_pending_spills
20165            .retain(|entry| entry.0 != anchor_vertex);
20166        self.spill_blocker_waiters.forget(anchor_vertex);
20167        if let Some(scope) = Self::structural_scope_from_cells(&prev_spill_cells) {
20168            self.record_structural_change(scope);
20169        }
20170        if let Some(scope) = Self::structural_scope_from_cells(targets) {
20171            self.record_structural_change(scope);
20172        }
20173
20174        if self.config.arrow_storage_enabled
20175            && self.config.delta_overlay_enabled
20176            && self.config.write_formula_overlay_enabled
20177        {
20178            if !prev_spill_cells.is_empty() {
20179                let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20180                    targets.iter().copied().collect();
20181                let empty = LiteralValue::Empty;
20182                for cell in prev_spill_cells.iter() {
20183                    if !target_set.contains(cell) {
20184                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20185                        self.mirror_value_to_computed_overlay(
20186                            &sheet_name,
20187                            cell.coord.row() + 1,
20188                            cell.coord.col() + 1,
20189                            &empty,
20190                        );
20191                    }
20192                }
20193            }
20194
20195            for (idx, cell) in targets.iter().enumerate() {
20196                if rows.is_empty() || rows[0].is_empty() {
20197                    break;
20198                }
20199                let width = rows[0].len();
20200                let r_off = idx / width;
20201                let c_off = idx % width;
20202                let v = rows[r_off][c_off].clone();
20203                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20204                self.mirror_value_to_computed_overlay(
20205                    &sheet_name,
20206                    cell.coord.row() + 1,
20207                    cell.coord.col() + 1,
20208                    &v,
20209                );
20210            }
20211        }
20212        Ok(())
20213    }
20214}
20215
20216#[cfg(test)]
20217#[path = "tests/authority_schedule_execution.rs"]
20218mod authority_schedule_execution;
20219
20220#[cfg(test)]
20221#[path = "tests/pending_spill.rs"]
20222mod pending_spill_tests;
20223
20224#[cfg(test)]
20225#[path = "tests/spill_batch_abort_192.rs"]
20226mod spill_batch_abort_192;
20227
20228// ── Effects pipeline (ticket 603) ──────────────────────────────────────────
20229//
20230// Compute → Plan → Apply separation for evaluation side-effects.
20231
20232use crate::engine::effects::Effect;
20233use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
20234
20235impl<R> Engine<R>
20236where
20237    R: EvaluationContext,
20238{
20239    /// Plan effects for a single vertex after its value has been computed.
20240    ///
20241    /// This reads graph state but only performs lightweight mutations
20242    /// (`set_kind`, `spill_mgr.reserve`) that are needed for correctness
20243    /// during the planning phase.  Value-changing mutations are deferred to
20244    /// `apply_effect`.
20245    pub(crate) fn plan_vertex_effects(
20246        &mut self,
20247        vertex_id: VertexId,
20248        computed_value: LiteralValue,
20249    ) -> Result<Vec<Effect>, ExcelError> {
20250        // FR3: a stale dynamic reader publishes nothing; FR2: anything else
20251        // leaves the dirty set at its commit (design §8.2).
20252        if self.freshness_armed() {
20253            if self.freshness_drop_stale(vertex_id) {
20254                return Ok(Vec::new());
20255            }
20256            let effects = self.plan_vertex_effects_unrecorded(vertex_id, computed_value)?;
20257            self.freshness_mark_committed(vertex_id);
20258            return Ok(effects);
20259        }
20260        self.plan_vertex_effects_unrecorded(vertex_id, computed_value)
20261    }
20262
20263    fn plan_vertex_effects_unrecorded(
20264        &mut self,
20265        vertex_id: VertexId,
20266        computed_value: LiteralValue,
20267    ) -> Result<Vec<Effect>, ExcelError> {
20268        let kind = self.graph.get_vertex_kind(vertex_id);
20269        let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
20270
20271        // If this vertex's cell is currently covered by a spill from a different
20272        // anchor, ignore the computed result.  Formula vertices are exempt:
20273        // they must still evaluate so that overlapping spills produce #SPILL!.
20274        if !is_formula {
20275            if let Some(cell) = self.graph.get_cell_ref(vertex_id)
20276                && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
20277                && owner != vertex_id
20278            {
20279                return Ok(Vec::new());
20280            }
20281            // Non-formula vertices: store value as-is (arrays remain arrays; no spill).
20282            return Ok(vec![Effect::WriteCell {
20283                vertex_id,
20284                value: computed_value,
20285            }]);
20286        }
20287
20288        match computed_value {
20289            LiteralValue::Array(rows) => self.plan_array_effects(vertex_id, rows),
20290            other => self.plan_scalar_effects(vertex_id, other),
20291        }
20292    }
20293
20294    /// Plan effects for a formula vertex that produced a scalar/error result.
20295    fn plan_scalar_effects(
20296        &mut self,
20297        vertex_id: VertexId,
20298        value: LiteralValue,
20299    ) -> Result<Vec<Effect>, ExcelError> {
20300        // Range admission substitutes the same cap error before allocating rows.
20301        // Preserve the owned-array rejection's kind, release and clear effects.
20302        if let LiteralValue::Error(error) = &value
20303            && error.kind == ExcelErrorKind::Spill
20304            && error.message.as_deref() == Some("SpillTooLarge")
20305            && let formualizer_common::ExcelErrorExtra::Spill {
20306                expected_rows,
20307                expected_cols,
20308            } = error.extra
20309            && u64::from(expected_rows).saturating_mul(u64::from(expected_cols))
20310                > u64::from(self.config.spill.max_spill_cells)
20311        {
20312            self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20313            return self.plan_spill_error_effects(
20314                vertex_id,
20315                "SpillTooLarge",
20316                expected_rows,
20317                expected_cols,
20318            );
20319        }
20320        if !matches!(&value, LiteralValue::Error(e) if e.kind == ExcelErrorKind::Spill) {
20321            self.blocked_pending_spills
20322                .retain(|entry| entry.0 != vertex_id);
20323            self.spill_blocker_waiters.forget(vertex_id);
20324        }
20325        let has_spill = self
20326            .graph
20327            .spill_cells_for_anchor(vertex_id)
20328            .is_some_and(|c| !c.is_empty());
20329
20330        let mut effects = Vec::new();
20331        if has_spill {
20332            effects.push(Effect::SpillClear {
20333                anchor_vertex: vertex_id,
20334            });
20335        }
20336        effects.push(Effect::WriteCell { vertex_id, value });
20337        Ok(effects)
20338    }
20339
20340    /// Plan effects for a formula vertex that produced an array result.
20341    fn plan_array_effects(
20342        &mut self,
20343        vertex_id: VertexId,
20344        rows: Vec<Vec<LiteralValue>>,
20345    ) -> Result<Vec<Effect>, ExcelError> {
20346        // Lightweight mutation needed for correct spill-blocking checks.
20347        self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20348
20349        let anchor = self
20350            .graph
20351            .get_cell_ref(vertex_id)
20352            .expect("cell ref for vertex");
20353        let sheet_id = anchor.sheet_id;
20354        let h = rows.len() as u32;
20355        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
20356
20357        // Hard cap to avoid vertex explosion from huge dynamic arrays.
20358        let spill_cells = (h as u64).saturating_mul(w as u64);
20359        if spill_cells > self.config.spill.max_spill_cells as u64 {
20360            return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
20361        }
20362
20363        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity).
20364        const PACKED_MAX_ROW: u32 = 1_048_575;
20365        const PACKED_MAX_COL: u32 = 16_383;
20366        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
20367        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
20368        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
20369            return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
20370        }
20371
20372        if h != 0 && w != 0 && self.graph.table_intersects_spill(anchor, end_row, end_col) {
20373            return self.plan_spill_error_effects(vertex_id, "Spill blocked by table", h, w);
20374        }
20375        let mut targets = Vec::new();
20376        for r in 0..h {
20377            for c in 0..w {
20378                targets.push(self.graph.make_cell_ref_internal(
20379                    sheet_id,
20380                    anchor.coord.row() + r,
20381                    anchor.coord.col() + c,
20382                ));
20383            }
20384        }
20385
20386        if h != 0 && w != 0 {
20387            let occupied = self.pending_spill_occupied(anchor, end_row, end_col);
20388            if occupied
20389                || self
20390                    .blocked_pending_spills
20391                    .iter()
20392                    .any(|entry| entry.0 == vertex_id)
20393            {
20394                self.spill_mgr.release_owner(vertex_id);
20395                self.remember_pending_spill(
20396                    vertex_id,
20397                    anchor,
20398                    Region::rect(
20399                        sheet_id,
20400                        anchor.coord.row(),
20401                        end_row,
20402                        anchor.coord.col(),
20403                        end_col,
20404                    ),
20405                )?;
20406            }
20407            if occupied {
20408                return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20409            }
20410        }
20411
20412        // Region lock via spill manager.
20413        match self.spill_mgr.reserve(
20414            vertex_id,
20415            anchor,
20416            SpillShape { rows: h, cols: w },
20417            SpillMeta {
20418                epoch: self.recalc_epoch,
20419                config: self.config.spill,
20420            },
20421        ) {
20422            Ok(()) => {
20423                // Validate spill region is available.
20424                let mut yield_effects = Vec::new();
20425                if let Err((e, blocker)) = self
20426                    .graph
20427                    .plan_spill_region_with_blocker(vertex_id, &targets)
20428                {
20429                    let yielding = if e.message.as_deref() == Some("BlockedBySpill") {
20430                        self.graph.spills_yielding_to(vertex_id, &targets)
20431                    } else {
20432                        None
20433                    };
20434                    if let Some(yielding) = yielding {
20435                        // A colliding spill later in (sheet, column, row)
20436                        // order yields: it is cleared and replans to
20437                        // `#SPILL!` with its dependents.
20438                        for &other in &yielding {
20439                            let (rows, cols) = self
20440                                .graph
20441                                .spill_extent_for_anchor(other)
20442                                .map(|(first, last)| {
20443                                    (
20444                                        last.coord.row() - first.coord.row() + 1,
20445                                        last.coord.col() - first.coord.col() + 1,
20446                                    )
20447                                })
20448                                .unwrap_or((0, 0));
20449                            yield_effects.push(Effect::SpillClear {
20450                                anchor_vertex: other,
20451                            });
20452                            yield_effects.push(Effect::WriteCell {
20453                                vertex_id: other,
20454                                value: Self::spill_error_value("Spill blocked", rows, cols),
20455                            });
20456                        }
20457                        self.graph.mark_dirty_many(&yielding);
20458                    } else {
20459                        // A formula or spill blocker can go away without this
20460                        // anchor recomputing: remember the attempted region so
20461                        // an edit inside it wakes the anchor.
20462                        if matches!(
20463                            e.message.as_deref(),
20464                            Some("BlockedByFormula" | "BlockedBySpill")
20465                        ) {
20466                            self.spill_blocker_waiters.wait(vertex_id, blocker);
20467                        }
20468                        return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20469                    }
20470                }
20471
20472                // Arrow-canonical mode: graph planning cannot see non-empty value blockers because
20473                // cell values are not cached in the dependency graph. Consult Arrow storage to
20474                // detect occupied cells in the target region.
20475                if !self.graph.value_cache_enabled() {
20476                    let sheet_name = self.graph.sheet_name(sheet_id);
20477                    if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
20478                        for cell in targets.iter() {
20479                            // Allow overwriting the anchor itself.
20480                            if *cell == anchor {
20481                                continue;
20482                            }
20483                            // Allow cells already owned by a spill (plan() validated spill ownership).
20484                            if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
20485                                continue;
20486                            }
20487                            // Skip formula blockers; plan() handled them (or allowed).
20488                            if let Some(vid) = self.graph.get_vertex_id_for_address(cell)
20489                                && vid != vertex_id
20490                            {
20491                                match self.graph.get_vertex_kind(vid) {
20492                                    VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20493                                        continue;
20494                                    }
20495                                    _ => {}
20496                                }
20497                            }
20498
20499                            let v = asheet.get_cell_value(
20500                                cell.coord.row() as usize,
20501                                cell.coord.col() as usize,
20502                            );
20503                            if !matches!(v, LiteralValue::Empty) {
20504                                return self.plan_spill_error_effects(
20505                                    vertex_id,
20506                                    "BlockedByValue",
20507                                    h,
20508                                    w,
20509                                );
20510                            }
20511                        }
20512                    }
20513                }
20514
20515                let top_left = rows
20516                    .first()
20517                    .and_then(|r| r.first())
20518                    .cloned()
20519                    .unwrap_or(LiteralValue::Empty);
20520
20521                let mut effects = yield_effects;
20522                // Clear previous spill if any.
20523                let has_prev = self
20524                    .graph
20525                    .spill_cells_for_anchor(vertex_id)
20526                    .is_some_and(|c| !c.is_empty());
20527                if has_prev {
20528                    effects.push(Effect::SpillClear {
20529                        anchor_vertex: vertex_id,
20530                    });
20531                }
20532                effects.push(Effect::SpillCommit {
20533                    anchor_vertex: vertex_id,
20534                    anchor_cell: anchor,
20535                    target_cells: targets,
20536                    values: rows,
20537                });
20538                effects.push(Effect::WriteCell {
20539                    vertex_id,
20540                    value: top_left,
20541                });
20542                Ok(effects)
20543            }
20544            Err(e) => {
20545                let msg = e.message.unwrap_or_else(|| "Spill blocked".to_string());
20546                self.plan_spill_error_effects(vertex_id, &msg, h, w)
20547            }
20548        }
20549    }
20550
20551    /// The `#SPILL!` value a blocked anchor holds.
20552    fn spill_error_value(message: &str, expected_rows: u32, expected_cols: u32) -> LiteralValue {
20553        LiteralValue::Error(
20554            ExcelError::new(ExcelErrorKind::Spill)
20555                .with_message(message)
20556                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20557                    expected_rows,
20558                    expected_cols,
20559                }),
20560        )
20561    }
20562
20563    /// Build the effect list for a spill that failed validation.
20564    fn plan_spill_error_effects(
20565        &mut self,
20566        vertex_id: VertexId,
20567        message: &str,
20568        expected_rows: u32,
20569        expected_cols: u32,
20570    ) -> Result<Vec<Effect>, ExcelError> {
20571        self.spill_mgr.release_owner(vertex_id);
20572        let spill_val = Self::spill_error_value(message, expected_rows, expected_cols);
20573
20574        let effects = vec![
20575            Effect::SpillClear {
20576                anchor_vertex: vertex_id,
20577            },
20578            Effect::WriteCell {
20579                vertex_id,
20580                value: spill_val,
20581            },
20582        ];
20583        Ok(effects)
20584    }
20585
20586    /// Apply a single effect, performing the actual graph mutations.
20587    pub(crate) fn apply_effect(
20588        &mut self,
20589        effect: &Effect,
20590        delta: Option<&mut DeltaCollector>,
20591        log: Option<&mut ChangeLog>,
20592    ) -> Result<(), ExcelError> {
20593        self.apply_effect_with_computed_writes(effect, delta, log, None)
20594    }
20595
20596    fn apply_effect_with_computed_writes(
20597        &mut self,
20598        effect: &Effect,
20599        delta: Option<&mut DeltaCollector>,
20600        log: Option<&mut ChangeLog>,
20601        computed_writes: Option<&mut ComputedWriteBuffer>,
20602    ) -> Result<(), ExcelError> {
20603        match effect {
20604            Effect::WriteCell { vertex_id, value } => {
20605                self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
20606            }
20607            Effect::SpillClear { anchor_vertex } => {
20608                self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
20609            }
20610            Effect::SpillCommit {
20611                anchor_vertex,
20612                anchor_cell: _,
20613                target_cells,
20614                values,
20615            } => {
20616                self.apply_spill_commit(
20617                    *anchor_vertex,
20618                    target_cells,
20619                    values.clone(),
20620                    delta,
20621                    log,
20622                    computed_writes,
20623                )?;
20624            }
20625        }
20626        Ok(())
20627    }
20628
20629    /// Apply a WriteCell effect.
20630    fn apply_write_cell(
20631        &mut self,
20632        vertex_id: VertexId,
20633        value: &LiteralValue,
20634        delta: Option<&mut DeltaCollector>,
20635        mut computed_writes: Option<&mut ComputedWriteBuffer>,
20636    ) -> Result<(), ExcelError> {
20637        if let Some(d) = delta
20638            && d.mode != DeltaMode::Off
20639        {
20640            if let Some(buffer) = computed_writes.as_deref_mut() {
20641                self.flush_computed_write_buffer(buffer)?;
20642            }
20643            if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
20644                let sheet_name = self.graph.sheet_name(cell.sheet_id);
20645                let old = self
20646                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20647                    .unwrap_or(LiteralValue::Empty);
20648                if old != *value {
20649                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20650                }
20651            }
20652        }
20653        self.graph.update_vertex_value_ref(vertex_id, value);
20654        self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
20655        Ok(())
20656    }
20657
20658    /// Apply a SpillClear effect.
20659    fn apply_spill_clear(
20660        &mut self,
20661        anchor_vertex: VertexId,
20662        delta: Option<&mut DeltaCollector>,
20663        log: Option<&mut ChangeLog>,
20664        computed_writes: Option<&mut ComputedWriteBuffer>,
20665    ) -> Result<(), ExcelError> {
20666        if let Some(buffer) = computed_writes {
20667            self.flush_computed_write_buffer(buffer)?;
20668        }
20669
20670        let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
20671        if spill_cells.is_empty() {
20672            return Ok(());
20673        }
20674
20675        // Snapshot for ChangeLog before clearing.
20676        let snapshot = if log.is_some() {
20677            self.snapshot_spill_for_anchor(anchor_vertex)
20678        } else {
20679            None
20680        };
20681
20682        // Record delta for cleared cells.
20683        if let Some(d) = delta
20684            && d.mode != DeltaMode::Off
20685        {
20686            let empty = LiteralValue::Empty;
20687            for cell in spill_cells.iter() {
20688                let sheet_name = self.graph.sheet_name(cell.sheet_id);
20689                let old = self
20690                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20691                    .unwrap_or(LiteralValue::Empty);
20692                if old != empty {
20693                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20694                }
20695            }
20696        }
20697
20698        self.graph.clear_spill_region(anchor_vertex);
20699        if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
20700            self.record_structural_change(scope);
20701        }
20702
20703        // Mirror Empty to Arrow overlay for cleared cells.
20704        if self.config.arrow_storage_enabled
20705            && self.config.delta_overlay_enabled
20706            && self.config.write_formula_overlay_enabled
20707        {
20708            let empty = LiteralValue::Empty;
20709            for cell in spill_cells.iter() {
20710                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20711                self.mirror_value_to_computed_overlay(
20712                    &sheet_name,
20713                    cell.coord.row() + 1,
20714                    cell.coord.col() + 1,
20715                    &empty,
20716                );
20717            }
20718        }
20719
20720        // ChangeLog.
20721        if let Some(log) = log
20722            && let Some(old) = snapshot
20723        {
20724            log.record(ChangeEvent::SpillCleared {
20725                anchor: anchor_vertex,
20726                old,
20727            });
20728        }
20729        Ok(())
20730    }
20731
20732    /// Apply a SpillCommit effect.
20733    fn apply_spill_commit(
20734        &mut self,
20735        anchor_vertex: VertexId,
20736        target_cells: &[CellRef],
20737        values: Vec<Vec<LiteralValue>>,
20738        delta: Option<&mut DeltaCollector>,
20739        log: Option<&mut ChangeLog>,
20740        computed_writes: Option<&mut ComputedWriteBuffer>,
20741    ) -> Result<(), ExcelError> {
20742        self.guard_pending_spill_commit(anchor_vertex, target_cells)?;
20743        if let Some(buffer) = computed_writes {
20744            self.flush_computed_write_buffer(buffer)?;
20745        }
20746
20747        // Snapshot for ChangeLog before commit.
20748        let old_snapshot = if log.is_some() {
20749            self.snapshot_spill_for_anchor(anchor_vertex)
20750        } else {
20751            None
20752        };
20753
20754        // Delegate to existing commit_spill_and_mirror for delta + overlay logic.
20755        self.commit_spill_and_mirror(anchor_vertex, target_cells, values.clone(), delta)?;
20756
20757        // ChangeLog.
20758        if let Some(log) = log {
20759            log.record(ChangeEvent::SpillCommitted {
20760                anchor: anchor_vertex,
20761                old: old_snapshot,
20762                new: SpillSnapshot {
20763                    target_cells: target_cells.to_vec(),
20764                    values,
20765                },
20766            });
20767        }
20768        Ok(())
20769    }
20770
20771    /// Snapshot a spill region for ChangeLog recording.
20772    ///
20773    /// Extracted from `VertexEditor::snapshot_spill_for_anchor` to be usable
20774    /// without creating a `VertexEditor`.
20775    fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
20776        let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
20777        if cells.is_empty() {
20778            return None;
20779        }
20780
20781        let max = self.config.spill.max_spill_cells as usize;
20782        let mut cells = cells;
20783        if cells.len() > max {
20784            cells.truncate(max);
20785        }
20786
20787        let first = *cells.first().expect("non-empty spill cells");
20788        let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
20789        let row0 = first.coord.row();
20790        let col0 = first.coord.col();
20791
20792        let mut max_row = row0;
20793        let mut max_col = col0;
20794        let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
20795        for cell in &cells {
20796            max_row = max_row.max(cell.coord.row());
20797            max_col = max_col.max(cell.coord.col());
20798            let v = self
20799                .get_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20800                .unwrap_or(LiteralValue::Empty);
20801            by_coord.insert((cell.coord.row(), cell.coord.col()), v);
20802        }
20803
20804        let rows = (max_row - row0 + 1) as usize;
20805        let cols = (max_col - col0 + 1) as usize;
20806        let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
20807        for r in 0..rows {
20808            let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
20809            for c in 0..cols {
20810                row.push(
20811                    by_coord
20812                        .get(&(row0 + r as u32, col0 + c as u32))
20813                        .cloned()
20814                        .unwrap_or(LiteralValue::Empty),
20815                );
20816            }
20817            values.push(row);
20818        }
20819
20820        Some(SpillSnapshot {
20821            target_cells: cells,
20822            values,
20823        })
20824    }
20825
20826    fn flush_before_range_dependent_vertex(
20827        &mut self,
20828        vertex_id: VertexId,
20829        computed_writes: &mut ComputedWriteBuffer,
20830    ) -> Result<(), ExcelError> {
20831        if self.graph.reads_compressed_range(vertex_id) {
20832            self.flush_computed_write_buffer(computed_writes)?;
20833        }
20834        Ok(())
20835    }
20836
20837    fn plan_vertex_effects_with_computed_flush(
20838        &mut self,
20839        vertex_id: VertexId,
20840        computed_value: LiteralValue,
20841        computed_writes: &mut ComputedWriteBuffer,
20842    ) -> Result<Vec<Effect>, ExcelError> {
20843        if matches!(&computed_value, LiteralValue::Array(_)) {
20844            self.flush_computed_write_buffer(computed_writes)?;
20845        }
20846        self.plan_vertex_effects(vertex_id, computed_value)
20847    }
20848
20849    // ── Layer evaluation via effects pipeline ──────────────────────────────
20850
20851    fn evaluate_small_layer_direct_effects(
20852        &mut self,
20853        layer: &super::scheduler::Layer,
20854        delta: Option<&mut DeltaCollector>,
20855        log: Option<&mut ChangeLog>,
20856        cancel_flag: Option<&AtomicBool>,
20857        cancel_check_every: usize,
20858        cancel_message: &'static str,
20859    ) -> Result<usize, ExcelError> {
20860        let cancel = cancel_flag.map(|flag| (flag, cancel_check_every, cancel_message));
20861        self.evaluate_layer_units(layer, delta, log, cancel, false)
20862    }
20863
20864    /// Sequential layer walk over its units (single cells and family runs):
20865    /// each unit evaluates, then its vertices' effects apply in order. With
20866    /// `buffered`, computed writes coalesce in a layer buffer (flushed before
20867    /// a unit that reads a compressed range, before array results, and at
20868    /// the end); otherwise they apply directly. `cancel` = (flag, check every
20869    /// N vertices, message).
20870    fn evaluate_layer_units(
20871        &mut self,
20872        layer: &super::scheduler::Layer,
20873        delta: Option<&mut DeltaCollector>,
20874        log: Option<&mut ChangeLog>,
20875        cancel: Option<(&AtomicBool, usize, &'static str)>,
20876        buffered: bool,
20877    ) -> Result<usize, ExcelError> {
20878        self.evaluate_layer_units_until(layer, delta, log, cancel, buffered, None)
20879    }
20880
20881    /// [`Self::evaluate_layer_units`] that stops before the next unit once
20882    /// `stop_at` has passed; returns the vertices evaluated (a prefix of
20883    /// the layer, all committed).
20884    fn evaluate_layer_units_until(
20885        &mut self,
20886        layer: &super::scheduler::Layer,
20887        mut delta: Option<&mut DeltaCollector>,
20888        mut log: Option<&mut ChangeLog>,
20889        cancel: Option<(&AtomicBool, usize, &'static str)>,
20890        buffered: bool,
20891        stop_at: Option<crate::instant::FzInstant>,
20892    ) -> Result<usize, ExcelError> {
20893        // A chain unit: its run through the chain lift, or else cell by
20894        // cell in row order, each written before the next reads it.
20895        let mut chain_values = None;
20896        if layer.sequential && !layer.runs.is_empty() {
20897            chain_values = match layer.runs.as_slice() {
20898                [run] if run.start == 0 && run.len as usize == layer.vertices.len() => {
20899                    self.try_chain_lift(*run, &layer.vertices)
20900                }
20901                _ => None,
20902            };
20903            if chain_values.is_none() {
20904                let cells = super::scheduler::Layer {
20905                    vertices: layer.vertices.clone(),
20906                    runs: Vec::new(),
20907                    sequential: true,
20908                };
20909                return self.evaluate_layer_units_until(&cells, delta, log, cancel, false, stop_at);
20910            }
20911        }
20912        let chained = chain_values.is_some();
20913        // The chain lift computed every member: one block write, as a run.
20914        let buffered = buffered || chained;
20915        // A dynamic reader's targets are not always ordered before it (its
20916        // pre-probe or observed reads can miss them, e.g. after a structural
20917        // edit). In a buffered layer a member's dirty flag is cleared at its
20918        // commit but its value written at the flush: the members committed
20919        // since the last flush count as dirty for the reader's freshness
20920        // check, which then re-plans it after them.
20921        let track_unflushed = buffered
20922            && self.freshness_armed()
20923            && layer.vertices.iter().any(|&v| self.graph.is_dynamic(v));
20924        let mut committed_unit: &[VertexId] = &[];
20925        let mut computed_writes = ComputedWriteBuffer::default();
20926        let mut next_check = 0usize;
20927        let mut done = 0usize;
20928        for unit in layer_units(layer) {
20929            if done > 0
20930                && let Some(stop_at) = stop_at
20931                && crate::instant::FzInstant::now() >= stop_at
20932            {
20933                self.flush_computed_write_buffer(&mut computed_writes)?;
20934                if track_unflushed {
20935                    self.freshness_flushed();
20936                }
20937                return Ok(done);
20938            }
20939            if let Some((flag, every, message)) = cancel
20940                && every > 0
20941                && done >= next_check
20942            {
20943                next_check = (done / every + 1) * every;
20944                if flag.load(Ordering::Relaxed) {
20945                    if buffered {
20946                        self.flush_computed_write_buffer(&mut computed_writes)?;
20947                    }
20948                    return Err(ExcelError::new(ExcelErrorKind::Cancelled)
20949                        .with_message(message.to_string()));
20950                }
20951            }
20952            if buffered && self.unit_reads_compressed_range(layer, unit) {
20953                self.flush_computed_write_buffer(&mut computed_writes)?;
20954            }
20955            // The previous unit's members are committed (dirty flags
20956            // cleared); while their values wait in the buffer, a dynamic
20957            // reader's read of them is stale (`freshness_dirty_reads`).
20958            if track_unflushed {
20959                if computed_writes.is_empty() {
20960                    self.freshness_flushed();
20961                } else {
20962                    self.freshness_note_unflushed(committed_unit);
20963                }
20964                committed_unit = unit_members(layer, unit);
20965            }
20966            let values = match (chain_values.take(), unit) {
20967                (Some(chain), LayerUnit::Run(run)) => {
20968                    if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
20969                        self.flush_computed_write_buffer(&mut computed_writes)?;
20970                        return Err(e);
20971                    }
20972                    let members =
20973                        &layer.vertices[run.start as usize..(run.start + run.len) as usize];
20974                    let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
20975                    match self.commit_run_numbers(
20976                        run,
20977                        members,
20978                        &chain,
20979                        delta_active,
20980                        Some(&mut computed_writes),
20981                    ) {
20982                        Ok(true) => {
20983                            done += chain.len();
20984                            continue;
20985                        }
20986                        Ok(false) => {}
20987                        Err(e) => {
20988                            self.flush_computed_write_buffer(&mut computed_writes)?;
20989                            return Err(e);
20990                        }
20991                    }
20992                    members
20993                        .iter()
20994                        .copied()
20995                        .zip(chain.into_iter().map(LiteralValue::Number))
20996                        .collect()
20997                }
20998                (_, unit) => self.evaluate_unit_immutable(layer, unit),
20999            };
21000            // Post-work boundary: the unit ran while (or after) the request
21001            // was cancelled; it is not committed and stays dirty.
21002            if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21003                self.flush_computed_write_buffer(&mut computed_writes)?;
21004                return Err(e);
21005            }
21006            done += values.len();
21007            if let LayerUnit::Run(run) = unit {
21008                let members = &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21009                let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21010                let committed = self.commit_run_scalars(
21011                    run,
21012                    members,
21013                    &values,
21014                    delta_active,
21015                    buffered.then_some(&mut computed_writes),
21016                );
21017                match committed {
21018                    Ok(true) => continue,
21019                    Ok(false) => {}
21020                    Err(e) => {
21021                        self.flush_computed_write_buffer(&mut computed_writes)?;
21022                        return Err(e);
21023                    }
21024                }
21025            }
21026            // A run unit's members were all evaluated before any commits
21027            // (FR5, FORM-192); a single cell is committed as it runs.
21028            let guarded = self.freshness_begin_batch_commit(&values);
21029            for (vertex_id, value) in values {
21030                let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21031                let effects = if buffered {
21032                    self.plan_vertex_effects_with_computed_flush(
21033                        vertex_id,
21034                        value,
21035                        &mut computed_writes,
21036                    )
21037                } else {
21038                    self.plan_vertex_effects(vertex_id, value)
21039                };
21040                let effects = match effects {
21041                    Ok(effects) => effects,
21042                    Err(e) => {
21043                        self.flush_computed_write_buffer(&mut computed_writes)?;
21044                        return Err(e);
21045                    }
21046                };
21047                for effect in &effects {
21048                    if let Err(e) = self.apply_effect_with_computed_writes(
21049                        effect,
21050                        delta.as_deref_mut(),
21051                        log.as_deref_mut(),
21052                        buffered.then_some(&mut computed_writes),
21053                    ) {
21054                        self.flush_computed_write_buffer(&mut computed_writes)?;
21055                        return Err(e);
21056                    }
21057                }
21058                self.freshness_keep_redirtied(redirtied, vertex_id);
21059            }
21060        }
21061        self.flush_computed_write_buffer(&mut computed_writes)?;
21062        if track_unflushed {
21063            self.freshness_flushed();
21064        }
21065        // Debug builds: every chain member equals the per-cell path, now
21066        // that the members above it are written.
21067        #[cfg(debug_assertions)]
21068        if chained {
21069            for &v in &layer.vertices {
21070                let cell = self.graph.get_cell_ref(v);
21071                let (sheet, row, col) = cell
21072                    .map(|c| {
21073                        (
21074                            self.graph.sheet_name(c.sheet_id).to_string(),
21075                            c.coord.row() + 1,
21076                            c.coord.col() + 1,
21077                        )
21078                    })
21079                    .expect("chain member cell");
21080                let written = self.get_cell_value(&sheet, row, col);
21081                let oracle = self
21082                    .evaluate_vertex_immutable(v)
21083                    .unwrap_or_else(LiteralValue::Error);
21084                assert!(
21085                    written
21086                        .as_ref()
21087                        .is_some_and(|w| same_value_bits(w, &oracle)),
21088                    "chain member {sheet}!R{row}C{col}: {written:?} vs per-cell {oracle:?}"
21089                );
21090            }
21091        }
21092        #[cfg(not(debug_assertions))]
21093        let _ = chained;
21094        Ok(layer.vertices.len())
21095    }
21096
21097    /// Evaluate a layer sequentially using the effects pipeline.
21098    fn evaluate_layer_sequential_effects(
21099        &mut self,
21100        layer: &super::scheduler::Layer,
21101    ) -> Result<usize, ExcelError> {
21102        let buffered = buffer_layer_writes(layer);
21103        self.evaluate_layer_units(layer, None, None, None, buffered)
21104    }
21105
21106    /// Evaluate a layer sequentially with delta collection via effects pipeline.
21107    fn evaluate_layer_sequential_with_delta_effects(
21108        &mut self,
21109        layer: &super::scheduler::Layer,
21110        delta: &mut DeltaCollector,
21111    ) -> Result<usize, ExcelError> {
21112        let buffered = buffer_layer_writes(layer);
21113        self.evaluate_layer_units(layer, Some(delta), None, None, buffered)
21114    }
21115
21116    /// Evaluate a layer sequentially with cancellation support via effects pipeline.
21117    fn evaluate_layer_sequential_cancellable_effects(
21118        &mut self,
21119        layer: &super::scheduler::Layer,
21120        cancel_flag: &AtomicBool,
21121    ) -> Result<usize, ExcelError> {
21122        let buffered = buffer_layer_writes(layer);
21123        let cancel = (cancel_flag, 256, "Evaluation cancelled within layer");
21124        self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21125    }
21126
21127    /// Evaluate a layer sequentially with more frequent cancellation for demand-driven eval.
21128    fn evaluate_layer_sequential_cancellable_demand_driven_effects(
21129        &mut self,
21130        layer: &super::scheduler::Layer,
21131        cancel_flag: &AtomicBool,
21132    ) -> Result<usize, ExcelError> {
21133        let buffered = buffer_layer_writes(layer);
21134        let cancel = (
21135            cancel_flag,
21136            128,
21137            "Demand-driven evaluation cancelled within layer",
21138        );
21139        self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21140    }
21141
21142    /// Evaluate a layer in parallel, applying via effects pipeline.
21143    fn evaluate_layer_parallel_effects(
21144        &mut self,
21145        layer: &super::scheduler::Layer,
21146        min_chunk: u32,
21147    ) -> Result<usize, ExcelError> {
21148        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21149
21150        let phases = self.parallel_phases(layer);
21151        let mut applied = 0usize;
21152
21153        for (units, group) in &phases {
21154            let group = &group[..];
21155            if group.is_empty() {
21156                continue;
21157            }
21158            let mut computed_writes = ComputedWriteBuffer::default();
21159
21160            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21161                thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, min_chunk));
21162
21163            // Post-work boundary: a group evaluated across a live
21164            // cancellation is not committed (it stays dirty).
21165            let results = results.and_then(|results| {
21166                self.live_cancellation_after_work(GROUP_CANCELLED)
21167                    .map(|()| results)
21168            });
21169            // FR3: a parallel group is one commit unit; one stale reader
21170            // drops the whole group (it stays dirty and replans).
21171            self.freshness_gate_group(group);
21172            match results {
21173                Ok(vertex_results) => {
21174                    let (vertex_results, committed) = self.commit_parallel_runs(
21175                        layer,
21176                        units,
21177                        vertex_results,
21178                        false,
21179                        &mut computed_writes,
21180                    )?;
21181                    applied = applied.saturating_add(committed);
21182                    // Arrays first, then scalars — establishes spill regions before
21183                    // scalar results that might land inside a spilled region.
21184                    // FR5 (FORM-192): members evaluated together must not
21185                    // clear a re-dirty from a spill committed before them.
21186                    let guarded = self.freshness_begin_batch_commit(&vertex_results);
21187                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21188                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21189                    for (vertex_id, result) in vertex_results {
21190                        if matches!(result, LiteralValue::Array(_)) {
21191                            arrays.push((vertex_id, result));
21192                        } else {
21193                            others.push((vertex_id, result));
21194                        }
21195                    }
21196                    for (vertex_id, result) in arrays {
21197                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21198                        let effects = match self.plan_vertex_effects_with_computed_flush(
21199                            vertex_id,
21200                            result,
21201                            &mut computed_writes,
21202                        ) {
21203                            Ok(effects) => effects,
21204                            Err(e) => {
21205                                self.flush_computed_write_buffer(&mut computed_writes)?;
21206                                return Err(e);
21207                            }
21208                        };
21209                        for effect in &effects {
21210                            if let Err(e) = self.apply_effect_with_computed_writes(
21211                                effect,
21212                                None,
21213                                None,
21214                                Some(&mut computed_writes),
21215                            ) {
21216                                self.flush_computed_write_buffer(&mut computed_writes)?;
21217                                return Err(e);
21218                            }
21219                        }
21220                        self.freshness_keep_redirtied(redirtied, vertex_id);
21221                        applied = applied.saturating_add(1);
21222                    }
21223                    // Make all array spill/top-left writes visible before scalar effects in this group.
21224                    self.flush_computed_write_buffer(&mut computed_writes)?;
21225                    for (vertex_id, result) in others {
21226                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21227                        let effects = match self.plan_vertex_effects_with_computed_flush(
21228                            vertex_id,
21229                            result,
21230                            &mut computed_writes,
21231                        ) {
21232                            Ok(effects) => effects,
21233                            Err(e) => {
21234                                self.flush_computed_write_buffer(&mut computed_writes)?;
21235                                return Err(e);
21236                            }
21237                        };
21238                        for effect in &effects {
21239                            if let Err(e) = self.apply_effect_with_computed_writes(
21240                                effect,
21241                                None,
21242                                None,
21243                                Some(&mut computed_writes),
21244                            ) {
21245                                self.flush_computed_write_buffer(&mut computed_writes)?;
21246                                return Err(e);
21247                            }
21248                        }
21249                        self.freshness_keep_redirtied(redirtied, vertex_id);
21250                        applied = applied.saturating_add(1);
21251                    }
21252                    // Flush at the group boundary; phase1 must be visible before phase2.
21253                    self.flush_computed_write_buffer(&mut computed_writes)?;
21254                }
21255                Err(e) => {
21256                    self.flush_computed_write_buffer(&mut computed_writes)?;
21257                    return Err(e);
21258                }
21259            }
21260        }
21261
21262        Ok(applied)
21263    }
21264
21265    /// Evaluate a layer in parallel with delta collection via effects pipeline.
21266    fn evaluate_layer_parallel_with_delta_effects(
21267        &mut self,
21268        layer: &super::scheduler::Layer,
21269        delta: &mut DeltaCollector,
21270    ) -> Result<usize, ExcelError> {
21271        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21272
21273        let phases = self.parallel_phases(layer);
21274        let mut applied = 0usize;
21275
21276        for (units, group) in &phases {
21277            let group = &group[..];
21278            if group.is_empty() {
21279                continue;
21280            }
21281            let mut computed_writes = ComputedWriteBuffer::default();
21282            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21283                thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, 8));
21284
21285            // Post-work boundary: a group evaluated across a live
21286            // cancellation is not committed (it stays dirty).
21287            let results = results.and_then(|results| {
21288                self.live_cancellation_after_work(GROUP_CANCELLED)
21289                    .map(|()| results)
21290            });
21291            // FR3: a parallel group is one commit unit; one stale reader
21292            // drops the whole group (it stays dirty and replans).
21293            self.freshness_gate_group(group);
21294            match results {
21295                Ok(vertex_results) => {
21296                    let (vertex_results, committed) = self.commit_parallel_runs(
21297                        layer,
21298                        units,
21299                        vertex_results,
21300                        delta.mode != DeltaMode::Off,
21301                        &mut computed_writes,
21302                    )?;
21303                    applied = applied.saturating_add(committed);
21304                    // FR5 (FORM-192): members evaluated together must not
21305                    // clear a re-dirty from a spill committed before them.
21306                    let guarded = self.freshness_begin_batch_commit(&vertex_results);
21307                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21308                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21309                    for (vertex_id, result) in vertex_results {
21310                        if matches!(result, LiteralValue::Array(_)) {
21311                            arrays.push((vertex_id, result));
21312                        } else {
21313                            others.push((vertex_id, result));
21314                        }
21315                    }
21316                    for (vertex_id, result) in arrays {
21317                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21318                        let effects = match self.plan_vertex_effects_with_computed_flush(
21319                            vertex_id,
21320                            result,
21321                            &mut computed_writes,
21322                        ) {
21323                            Ok(effects) => effects,
21324                            Err(e) => {
21325                                self.flush_computed_write_buffer(&mut computed_writes)?;
21326                                return Err(e);
21327                            }
21328                        };
21329                        for effect in &effects {
21330                            if let Err(e) = self.apply_effect_with_computed_writes(
21331                                effect,
21332                                Some(delta),
21333                                None,
21334                                Some(&mut computed_writes),
21335                            ) {
21336                                self.flush_computed_write_buffer(&mut computed_writes)?;
21337                                return Err(e);
21338                            }
21339                        }
21340                        self.freshness_keep_redirtied(redirtied, vertex_id);
21341                        applied = applied.saturating_add(1);
21342                    }
21343                    self.flush_computed_write_buffer(&mut computed_writes)?;
21344                    for (vertex_id, result) in others {
21345                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21346                        let effects = match self.plan_vertex_effects_with_computed_flush(
21347                            vertex_id,
21348                            result,
21349                            &mut computed_writes,
21350                        ) {
21351                            Ok(effects) => effects,
21352                            Err(e) => {
21353                                self.flush_computed_write_buffer(&mut computed_writes)?;
21354                                return Err(e);
21355                            }
21356                        };
21357                        for effect in &effects {
21358                            if let Err(e) = self.apply_effect_with_computed_writes(
21359                                effect,
21360                                Some(delta),
21361                                None,
21362                                Some(&mut computed_writes),
21363                            ) {
21364                                self.flush_computed_write_buffer(&mut computed_writes)?;
21365                                return Err(e);
21366                            }
21367                        }
21368                        self.freshness_keep_redirtied(redirtied, vertex_id);
21369                        applied = applied.saturating_add(1);
21370                    }
21371                    self.flush_computed_write_buffer(&mut computed_writes)?;
21372                }
21373                Err(e) => {
21374                    self.flush_computed_write_buffer(&mut computed_writes)?;
21375                    return Err(e);
21376                }
21377            }
21378        }
21379
21380        Ok(applied)
21381    }
21382
21383    /// Evaluate a layer in parallel with cancellation support via effects pipeline.
21384    fn evaluate_layer_parallel_cancellable_effects(
21385        &mut self,
21386        layer: &super::scheduler::Layer,
21387        cancel_flag: &AtomicBool,
21388    ) -> Result<usize, ExcelError> {
21389        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21390
21391        if cancel_flag.load(Ordering::Relaxed) {
21392            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21393                .with_message("Parallel evaluation cancelled before starting".to_string()));
21394        }
21395
21396        let phases = self.parallel_phases(layer);
21397        let mut applied = 0usize;
21398
21399        for (units, group) in &phases {
21400            let group = &group[..];
21401            if group.is_empty() {
21402                continue;
21403            }
21404            let mut computed_writes = ComputedWriteBuffer::default();
21405
21406            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> = thread_pool
21407                .install(|| self.evaluate_units_parallel(layer, units, Some(cancel_flag), 8));
21408
21409            // Post-work boundary: a group evaluated across a live
21410            // cancellation is not committed (it stays dirty).
21411            let results = results.and_then(|results| {
21412                self.live_cancellation_after_work(GROUP_CANCELLED)
21413                    .map(|()| results)
21414            });
21415            // FR3: a parallel group is one commit unit; one stale reader
21416            // drops the whole group (it stays dirty and replans).
21417            self.freshness_gate_group(group);
21418            match results {
21419                Ok(vertex_results) => {
21420                    let (vertex_results, committed) = self.commit_parallel_runs(
21421                        layer,
21422                        units,
21423                        vertex_results,
21424                        false,
21425                        &mut computed_writes,
21426                    )?;
21427                    applied = applied.saturating_add(committed);
21428                    // FR5 (FORM-192): members evaluated together must not
21429                    // clear a re-dirty from a spill committed before them.
21430                    let guarded = self.freshness_begin_batch_commit(&vertex_results);
21431                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21432                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21433                    for (vertex_id, result) in vertex_results {
21434                        if matches!(result, LiteralValue::Array(_)) {
21435                            arrays.push((vertex_id, result));
21436                        } else {
21437                            others.push((vertex_id, result));
21438                        }
21439                    }
21440                    for (vertex_id, result) in arrays {
21441                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21442                        let effects = match self.plan_vertex_effects_with_computed_flush(
21443                            vertex_id,
21444                            result,
21445                            &mut computed_writes,
21446                        ) {
21447                            Ok(effects) => effects,
21448                            Err(e) => {
21449                                self.flush_computed_write_buffer(&mut computed_writes)?;
21450                                return Err(e);
21451                            }
21452                        };
21453                        for effect in &effects {
21454                            if let Err(e) = self.apply_effect_with_computed_writes(
21455                                effect,
21456                                None,
21457                                None,
21458                                Some(&mut computed_writes),
21459                            ) {
21460                                self.flush_computed_write_buffer(&mut computed_writes)?;
21461                                return Err(e);
21462                            }
21463                        }
21464                        self.freshness_keep_redirtied(redirtied, vertex_id);
21465                        applied = applied.saturating_add(1);
21466                    }
21467                    self.flush_computed_write_buffer(&mut computed_writes)?;
21468                    for (vertex_id, result) in others {
21469                        let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21470                        let effects = match self.plan_vertex_effects_with_computed_flush(
21471                            vertex_id,
21472                            result,
21473                            &mut computed_writes,
21474                        ) {
21475                            Ok(effects) => effects,
21476                            Err(e) => {
21477                                self.flush_computed_write_buffer(&mut computed_writes)?;
21478                                return Err(e);
21479                            }
21480                        };
21481                        for effect in &effects {
21482                            if let Err(e) = self.apply_effect_with_computed_writes(
21483                                effect,
21484                                None,
21485                                None,
21486                                Some(&mut computed_writes),
21487                            ) {
21488                                self.flush_computed_write_buffer(&mut computed_writes)?;
21489                                return Err(e);
21490                            }
21491                        }
21492                        self.freshness_keep_redirtied(redirtied, vertex_id);
21493                        applied = applied.saturating_add(1);
21494                    }
21495                    self.flush_computed_write_buffer(&mut computed_writes)?;
21496                }
21497                Err(e) => {
21498                    self.flush_computed_write_buffer(&mut computed_writes)?;
21499                    return Err(e);
21500                }
21501            }
21502        }
21503
21504        Ok(applied)
21505    }
21506
21507    // ── Top-level evaluate_all_logged ───────────────────────────────────────
21508
21509    /// Evaluate all dirty/volatile vertices, recording effects into a ChangeLog.
21510    ///
21511    /// This is the same flow as `evaluate_all` but threads a ChangeLog through
21512    /// every effect application so that spill commits/clears are captured.
21513    pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
21514        self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
21515            engine.evaluate_all_logged_unobserved(log)
21516        })
21517    }
21518
21519    fn evaluate_all_logged_unobserved(
21520        &mut self,
21521        log: &mut ChangeLog,
21522    ) -> Result<EvalResult, ExcelError> {
21523        self.observe_function_semantic_epoch()?;
21524        let _source_cache = self.source_cache_session();
21525        self.validate_deterministic_mode()?;
21526        if self.config.defer_graph_building {
21527            self.build_graph_all()?;
21528        }
21529        self.require_unified_authority()?;
21530        self.begin_evaluation_request();
21531        self.reset_virtual_dep_telemetry_if_disabled();
21532        let start = crate::instant::FzInstant::now();
21533        let mut computed_vertices = 0;
21534        let mut cycle_errors = 0;
21535
21536        let mut replan_iterations = 0;
21537        const MAX_REPLAN: usize = 5;
21538        let mut telemetry = self
21539            .config
21540            .enable_virtual_dep_telemetry
21541            .then(|| self.start_virtual_dep_telemetry());
21542
21543        log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
21544
21545        let result = (|| -> Result<EvalResult, ExcelError> {
21546            loop {
21547                let to_evaluate = self.graph.get_evaluation_vertices();
21548                if to_evaluate.is_empty() {
21549                    if let Some(t) = telemetry.as_mut()
21550                        && t.bailout_reason.is_none()
21551                    {
21552                        t.bailout_reason = Some("no_work");
21553                    }
21554                    break;
21555                }
21556
21557                let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21558                if let Some(t) = telemetry.as_mut() {
21559                    Self::accumulate_schedule_meta(t, &meta);
21560                }
21561
21562                // Walk units in condensation order: stamp cycles at their
21563                // position, evaluate layers with ChangeLog recording.
21564                self.begin_pass(&schedule);
21565                for (unit_index, &unit) in schedule.units.iter().enumerate() {
21566                    match unit {
21567                        ScheduleUnit::Cycle(i) => {
21568                            // Journal integration (design doc §4 last row): the
21569                            // ChangeLog in this path only records SpillClear /
21570                            // SpillCommit events; WriteCell effects are never
21571                            // logged (see `apply_write_cell`). Runtime SCC tasks
21572                            // write values directly and never spill (§7.9 stamps
21573                            // would-be anchors), and their spill *teardown* is the
21574                            // same unlogged `stamp_cycle_error` the Static path
21575                            // already uses here — so direct commits coexist with
21576                            // the journal cleanly, with identical semantics to
21577                            // Static. Pinned by `scc_runtime_cycles` tests.
21578                            if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
21579                            {
21580                                cycle_errors += 1;
21581                            }
21582                        }
21583                        ScheduleUnit::Layer(i) => {
21584                            computed_vertices +=
21585                                self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
21586                        }
21587                    }
21588                    if self.stop_after_unit(&schedule, unit_index) {
21589                        break;
21590                    }
21591                }
21592
21593                let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21594                if let Some(t) = telemetry.as_mut() {
21595                    t.changed_vdeps_total += changed_vertices.len();
21596                }
21597                self.resource_checkpoint(0)?;
21598                if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
21599                    if let Some(t) = telemetry.as_mut() {
21600                        t.bailout_reason = Some("converged");
21601                    }
21602                    break;
21603                }
21604                if replan_iterations >= MAX_REPLAN {
21605                    if let Some(mut t) = telemetry.take() {
21606                        t.bailout_reason = Some("max_replan");
21607                        t.replan_iterations = replan_iterations;
21608                        self.last_virtual_dep_telemetry = t;
21609                    }
21610                    return Err(
21611                        self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21612                    );
21613                }
21614                replan_iterations += 1;
21615            }
21616
21617            if let Some(mut t) = telemetry {
21618                t.replan_iterations = replan_iterations;
21619                self.last_virtual_dep_telemetry = t;
21620            }
21621
21622            self.redirty_for_next_recalc();
21623            self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21624
21625            Ok(EvalResult {
21626                computed_vertices,
21627                cycle_errors,
21628                elapsed: start.elapsed(),
21629            })
21630        })();
21631        log.end_compound();
21632        result
21633    }
21634
21635    /// Evaluate a single layer with ChangeLog recording.
21636    fn evaluate_layer_logged(
21637        &mut self,
21638        layer: &super::scheduler::Layer,
21639        log: &mut ChangeLog,
21640    ) -> Result<usize, ExcelError> {
21641        self.resource_checkpoint(layer.vertices.len() as u64)?;
21642        self.evaluate_layer_units(layer, None, Some(log), None, true)
21643    }
21644}