formualizer_eval/engine/
virtual_deps.rs

1use crate::engine::VertexId;
2use crate::engine::VertexKind;
3use crate::engine::eval::Engine;
4use crate::engine::template::region::Region;
5use crate::engine::used_extent::{
6    ExtentPolicy, OpenRangeBounds, ResolvedExtent, resolve_used_extent_with_fallback,
7};
8use crate::traits::{
9    EvaluationContext, FunctionProvider, NamedRangeResolver, Range, RangeResolver,
10    ReferenceResolver, Resolver, SourceResolver, Table, TableResolver,
11};
12use formualizer_common::{ExcelError, LiteralValue};
13use formualizer_parse::parser::{ReferenceType, TableReference};
14use rustc_hash::FxHashSet;
15use std::sync::Mutex;
16
17use crate::interpreter::Interpreter;
18
19pub struct DynamicRefCollector<'a, R: EvaluationContext> {
20    pub engine: &'a Engine<R>,
21    pub current_sheet: &'a str,
22    pub(crate) collected: Mutex<FxHashSet<VertexId>>,
23    pub(crate) collected_regions: Mutex<FxHashSet<Region>>,
24}
25
26impl<'a, R: EvaluationContext> DynamicRefCollector<'a, R> {
27    pub fn new(engine: &'a Engine<R>, current_sheet: &'a str) -> Self {
28        Self {
29            engine,
30            current_sheet,
31            collected: Mutex::new(FxHashSet::default()),
32            collected_regions: Mutex::new(FxHashSet::default()),
33        }
34    }
35
36    fn collect_formula_vertices_in_rect(
37        &self,
38        sheet_name: &str,
39        sr: u32,
40        sc: u32,
41        er: u32,
42        ec: u32,
43    ) {
44        let Some(sheet_id) = self.engine.graph.sheet_id(sheet_name) else {
45            return;
46        };
47        let sr0 = sr.saturating_sub(1);
48        let er0 = er.saturating_sub(1);
49        let sc0 = sc.saturating_sub(1);
50        let ec0 = ec.saturating_sub(1);
51        self.collected_regions
52            .lock()
53            .unwrap()
54            .insert(Region::rect(sheet_id, sr0, er0, sc0, ec0).normalized());
55        if self.engine.graph.sheet_index(sheet_id).is_none() {
56            return;
57        }
58
59        let mut out = self.collected.lock().unwrap();
60        for u in self.engine.graph.vertices_in_cols(sheet_id, sc0, ec0) {
61            let Some(row0) = self.engine.graph.vertex_grid_addr(u).map(|addr| addr.row()) else {
62                continue;
63            };
64            if row0 < sr0 || row0 > er0 {
65                continue;
66            }
67            match self.engine.graph.get_vertex_kind(u) {
68                VertexKind::FormulaScalar | VertexKind::FormulaArray
69                    if (self.engine.graph.is_dirty(u) || self.engine.graph.is_volatile(u)) =>
70                {
71                    out.insert(u);
72                }
73                _ => {}
74            }
75        }
76    }
77
78    fn collect_formula_vertices_for_range(
79        &self,
80        sheet_name: &str,
81        start_row: Option<u32>,
82        start_col: Option<u32>,
83        end_row: Option<u32>,
84        end_col: Option<u32>,
85    ) {
86        let Some(extent) = resolve_used_extent_with_fallback(
87            OpenRangeBounds {
88                start_row,
89                start_column: start_col,
90                end_row,
91                end_column: end_col,
92            },
93            ExtentPolicy::EvaluationCompat {
94                fallback_row: None,
95                fallback_column: None,
96            },
97            || {
98                self.engine
99                    .sheet_bounds(sheet_name)
100                    .map(|_| self.engine.config.max_open_ended_rows)
101            },
102            || {
103                self.engine
104                    .sheet_bounds(sheet_name)
105                    .map(|_| self.engine.config.max_open_ended_cols)
106            },
107            |first, last| self.engine.used_rows_for_columns(sheet_name, first, last),
108            |first, last| self.engine.used_cols_for_rows(sheet_name, first, last),
109        ) else {
110            return;
111        };
112
113        self.collect_formula_vertices_in_rect(
114            sheet_name,
115            extent.start_row,
116            extent.start_column,
117            extent.end_row,
118            extent.end_column,
119        );
120    }
121}
122
123impl<'a, R: EvaluationContext> ReferenceResolver for DynamicRefCollector<'a, R> {
124    fn resolve_cell_reference(
125        &self,
126        sheet: Option<&str>,
127        row: u32,
128        col: u32,
129    ) -> Result<LiteralValue, ExcelError> {
130        let sheet_name = sheet.unwrap_or(self.current_sheet);
131        if let Some(sheet_id) = self.engine.graph.sheet_id(sheet_name) {
132            self.collected_regions.lock().unwrap().insert(Region::point(
133                sheet_id,
134                row.saturating_sub(1),
135                col.saturating_sub(1),
136            ));
137        }
138        if let Some(vid) = self
139            .engine
140            .graph
141            .get_vertex_id_for_address(&self.engine.graph.make_cell_ref(sheet_name, row, col))
142        {
143            self.collected.lock().unwrap().insert(vid);
144        }
145        self.engine.resolve_cell_reference(sheet, row, col)
146    }
147}
148
149impl<'a, R: EvaluationContext> RangeResolver for DynamicRefCollector<'a, R> {
150    fn resolve_range_reference(
151        &self,
152        sheet: Option<&str>,
153        sr: Option<u32>,
154        sc: Option<u32>,
155        er: Option<u32>,
156        ec: Option<u32>,
157    ) -> Result<Box<dyn Range>, ExcelError> {
158        let sheet_name = sheet.unwrap_or(self.current_sheet);
159        self.collect_formula_vertices_for_range(sheet_name, sr, sc, er, ec);
160        self.engine.resolve_range_reference(sheet, sr, sc, er, ec)
161    }
162}
163
164impl<'a, R: EvaluationContext> NamedRangeResolver for DynamicRefCollector<'a, R> {
165    fn resolve_named_range_reference(
166        &self,
167        name: &str,
168    ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
169        self.engine.resolve_named_range_reference(name)
170    }
171}
172
173impl<'a, R: EvaluationContext> TableResolver for DynamicRefCollector<'a, R> {
174    fn resolve_table_reference(&self, tref: &TableReference) -> Result<Box<dyn Table>, ExcelError> {
175        self.engine.resolve_table_reference(tref)
176    }
177}
178
179impl<'a, R: EvaluationContext> SourceResolver for DynamicRefCollector<'a, R> {
180    fn source_scalar_version(&self, name: &str) -> Option<u64> {
181        self.engine.source_scalar_version(name)
182    }
183    fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
184        self.engine.resolve_source_scalar(name)
185    }
186    fn source_table_version(&self, name: &str) -> Option<u64> {
187        self.engine.source_table_version(name)
188    }
189    fn resolve_source_table(&self, name: &str) -> Result<Box<dyn Table>, ExcelError> {
190        self.engine.resolve_source_table(name)
191    }
192}
193
194impl<'a, R: EvaluationContext> Resolver for DynamicRefCollector<'a, R> {}
195
196impl<'a, R: EvaluationContext> FunctionProvider for DynamicRefCollector<'a, R> {
197    fn planning_semantic_revision(&self) -> Option<u64> {
198        self.engine.planning_semantic_revision()
199    }
200
201    fn get_function(
202        &self,
203        ns: &str,
204        name: &str,
205    ) -> Option<std::sync::Arc<dyn crate::traits::Function>> {
206        self.engine.get_function(ns, name)
207    }
208
209    fn get_function_for_planning(
210        &self,
211        ns: &str,
212        name: &str,
213    ) -> Option<std::sync::Arc<dyn crate::traits::Function>> {
214        self.engine.get_function_for_planning(ns, name)
215    }
216}
217
218impl<'a, R: EvaluationContext> EvaluationContext for DynamicRefCollector<'a, R> {
219    fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
220        self.engine.cancellation_token()
221    }
222
223    fn materialized_array_cells(&self) -> u64 {
224        self.engine.materialized_array_cells()
225    }
226
227    fn resolve_cell_format(
228        &self,
229        sheet: Option<&str>,
230        row: u32,
231        col: u32,
232        current_sheet: &str,
233    ) -> Option<crate::format::FormatId> {
234        self.engine
235            .resolve_cell_format(sheet, row, col, current_sheet)
236    }
237
238    fn format_class(
239        &self,
240        format: crate::format::FormatId,
241    ) -> Option<formualizer_common::numfmt::FormatClass> {
242        self.engine.format_class(format)
243    }
244
245    fn record_cell_derived_format(
246        &self,
247        sheet: &str,
248        row: u32,
249        col: u32,
250        format: Option<crate::format::FormatId>,
251    ) {
252        self.engine
253            .record_cell_derived_format(sheet, row, col, format)
254    }
255
256    fn resolve_range_view<'c>(
257        &'c self,
258        reference: &ReferenceType,
259        current_sheet: &str,
260    ) -> Result<crate::engine::range_view::RangeView<'c>, ExcelError> {
261        // Collect vertices directly
262        match reference {
263            ReferenceType::Cell {
264                sheet, row, col, ..
265            } => {
266                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
267                self.collect_formula_vertices_in_rect(sheet_name, *row, *col, *row, *col);
268            }
269            ReferenceType::Range {
270                sheet,
271                start_row,
272                start_col,
273                end_row,
274                end_col,
275                ..
276            } => {
277                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
278                self.collect_formula_vertices_for_range(
279                    sheet_name, *start_row, *start_col, *end_row, *end_col,
280                );
281            }
282            ReferenceType::NamedRange(name) => {
283                let sid = self.engine.sheet_id(current_sheet);
284                if let Some(s) = sid
285                    && let Some(nr) = self.engine.graph.resolve_name_entry(name, s)
286                {
287                    let vid = nr.vertex;
288                    self.collected.lock().unwrap().insert(vid);
289                }
290            }
291            ReferenceType::Table(_) => {
292                // Table references might be tricky, skip for now or resolve from graph if possible
293            }
294            _ => {}
295        }
296
297        self.engine.resolve_range_view(reference, current_sheet)
298    }
299
300    fn resolve_spill_reference(
301        &self,
302        anchor: &ReferenceType,
303        current_sheet: &str,
304    ) -> Result<ReferenceType, ExcelError> {
305        // The anchor is a dependency whether or not it currently spills.
306        match anchor {
307            ReferenceType::Cell {
308                sheet, row, col, ..
309            } => {
310                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
311                self.collect_formula_vertices_in_rect(sheet_name, *row, *col, *row, *col);
312            }
313            ReferenceType::NamedRange(name) => {
314                if let Some(sheet_id) = self.engine.sheet_id(current_sheet)
315                    && let Some(named) = self.engine.graph.resolve_name_entry(name, sheet_id)
316                {
317                    self.collected.lock().unwrap().insert(named.vertex);
318                }
319            }
320            _ => {}
321        }
322        self.engine.resolve_spill_reference(anchor, current_sheet)
323    }
324}
325
326pub struct RangeVirtualDepProvider;
327
328impl RangeVirtualDepProvider {
329    pub(crate) fn resolve_range<R: EvaluationContext>(
330        engine: &Engine<R>,
331        sheet_name: &str,
332        range: &formualizer_common::SheetRangeRef<'_>,
333    ) -> Option<ResolvedExtent> {
334        resolve_used_extent_with_fallback(
335            OpenRangeBounds {
336                start_row: range.start_row.map(|bound| bound.index + 1),
337                start_column: range.start_col.map(|bound| bound.index + 1),
338                end_row: range.end_row.map(|bound| bound.index + 1),
339                end_column: range.end_col.map(|bound| bound.index + 1),
340            },
341            ExtentPolicy::VirtualDependencyCompat {
342                fallback_row: None,
343                fallback_column: None,
344            },
345            || {
346                engine
347                    .sheet_bounds(sheet_name)
348                    .map(|_| engine.config.max_open_ended_rows)
349            },
350            || {
351                engine
352                    .sheet_bounds(sheet_name)
353                    .map(|_| engine.config.max_open_ended_cols)
354            },
355            |first, last| engine.used_rows_for_columns(sheet_name, first, last),
356            |first, last| engine.used_cols_for_rows(sheet_name, first, last),
357        )
358    }
359
360    #[cfg(any(test, feature = "legacy_oracle"))]
361    pub fn get_virtual_deps<R: EvaluationContext>(
362        engine: &Engine<R>,
363        v: VertexId,
364    ) -> Vec<VertexId> {
365        let mut deps = Vec::new();
366        if let Some(ranges) = engine.graph.get_range_dependencies(v) {
367            let current_sheet_id = engine.graph.get_vertex_sheet_id(v);
368            for r in ranges {
369                let sheet_id = match r.sheet {
370                    formualizer_common::SheetLocator::Id(id) => id,
371                    _ => current_sheet_id,
372                };
373                let sheet_name = engine.graph.sheet_name(sheet_id);
374
375                let Some(extent) = Self::resolve_range(engine, sheet_name, r) else {
376                    continue;
377                };
378                let sr = extent.start_row;
379                let sc = extent.start_column;
380                let er = extent.end_row;
381                let ec = extent.end_column;
382
383                if engine.graph.sheet_index(sheet_id).is_some() {
384                    let sr0 = sr.saturating_sub(1);
385                    let er0 = er.saturating_sub(1);
386                    let sc0 = sc.saturating_sub(1);
387                    let ec0 = ec.saturating_sub(1);
388                    for u in engine.graph.vertices_in_cols(sheet_id, sc0, ec0) {
389                        let Some(pc) = engine.graph.vertex_grid_addr(u) else {
390                            continue;
391                        };
392                        let row0 = pc.row();
393                        if row0 < sr0 || row0 > er0 {
394                            continue;
395                        }
396                        match engine.graph.get_vertex_kind(u) {
397                            VertexKind::FormulaScalar | VertexKind::FormulaArray
398                                if (engine.graph.is_dirty(u) || engine.graph.is_volatile(u))
399                                    && u != v =>
400                            {
401                                deps.push(u);
402                            }
403                            _ => {}
404                        }
405                    }
406                }
407            }
408        }
409        deps
410    }
411}
412
413pub struct VirtualDepBuilder<'a, R: EvaluationContext> {
414    engine: &'a Engine<R>,
415}
416
417impl<'a, R: EvaluationContext> VirtualDepBuilder<'a, R> {
418    pub fn new(engine: &'a Engine<R>) -> Self {
419        Self { engine }
420    }
421    /// Plan hints for `candidates`. Under `unified_authority` a compressed
422    /// range read is an ordinary static edge of the relation (R-1), so the
423    /// planner already orders every formula inside the range before its
424    /// reader; only dynamic readers get hints (design §8.2). Enumerating the
425    /// range members costs |readers| × |range formulas| hints (100M for
426    /// 1,000 SUMIFS over a 100k formula column), and resolving their used
427    /// extent at plan time caches it before same-pass spills commit.
428    pub fn build(
429        &self,
430        candidates: &[VertexId],
431    ) -> (
432        rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
433        Vec<VertexId>,
434    ) {
435        self.build_inner(candidates, false)
436    }
437
438    /// Legacy's hints, range members included: what the legacy scheduler
439    /// (the test oracle) needs, since its range stripes carry no edges.
440    pub fn build_with_range_members(
441        &self,
442        candidates: &[VertexId],
443    ) -> (
444        rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
445        Vec<VertexId>,
446    ) {
447        self.build_inner(candidates, true)
448    }
449
450    fn build_inner(
451        &self,
452        candidates: &[VertexId],
453        range_members: bool,
454    ) -> (
455        rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
456        Vec<VertexId>,
457    ) {
458        let mut vdeps: rustc_hash::FxHashMap<VertexId, Vec<VertexId>> =
459            rustc_hash::FxHashMap::default();
460        let augmented_vertices: Vec<VertexId> = Vec::new(); // Will be populated in Phase 3
461
462        for &v in candidates {
463            // Range members are legacy's hints (its stripes carry no
464            // scheduling edges): the oracle scheduler's input only.
465            #[cfg(any(test, feature = "legacy_oracle"))]
466            let mut deps = if range_members {
467                RangeVirtualDepProvider::get_virtual_deps(self.engine, v)
468            } else {
469                Vec::new()
470            };
471            #[cfg(not(any(test, feature = "legacy_oracle")))]
472            let mut deps = Vec::new();
473            // Under the authority a reader with an observed read set is
474            // planned from it (rdi_dyn, rectangle hints); the pre-probe is
475            // for first evaluations only (design §8.2).
476            let observed =
477                !range_members && self.engine.graph.authority_host().observed(v).is_some();
478            let dynamic_deps = if observed {
479                Vec::new()
480            } else {
481                DynamicRefVirtualDepProvider::get_virtual_deps(self.engine, v)
482            };
483
484            deps.extend(dynamic_deps);
485            deps.sort_unstable();
486            deps.dedup();
487
488            if !deps.is_empty() {
489                vdeps.insert(v, deps);
490            }
491        }
492
493        (vdeps, augmented_vertices)
494    }
495}
496
497pub struct DynamicRefVirtualDepProvider;
498
499impl DynamicRefVirtualDepProvider {
500    fn collect<R: EvaluationContext>(
501        engine: &Engine<R>,
502        v: VertexId,
503    ) -> (Vec<VertexId>, Vec<Region>) {
504        if !engine.graph.is_dynamic(v) {
505            return (Vec::new(), Vec::new());
506        }
507        let Some(view) = engine.graph.formula_view(v) else {
508            return (Vec::new(), Vec::new());
509        };
510        let sheet_id = engine.graph.get_vertex_sheet_id(v);
511        let sheet_name = engine.graph.sheet_name(sheet_id);
512        let collector = DynamicRefCollector::new(engine, sheet_name);
513        let cell_ref = engine
514            .graph
515            .get_cell_ref(v)
516            .unwrap_or_else(|| engine.graph.make_cell_ref(sheet_name, 0, 0));
517        // This query is public and may run outside an evaluation request, when
518        // the per-request function table can be empty or stale: use it only
519        // when it is current, otherwise resolve through the provider.
520        let interpreter = Interpreter::new_with_cell(&collector, sheet_name, cell_ref);
521        let interpreter = match engine.current_resolved_functions() {
522            Some(table) => interpreter.with_resolved_functions(table),
523            None => interpreter,
524        };
525        let _ = interpreter.evaluate_formula_view(
526            view,
527            engine.graph.data_store(),
528            engine.graph.sheet_reg(),
529        );
530        let mut deps = collector
531            .collected
532            .lock()
533            .unwrap()
534            .iter()
535            .copied()
536            .filter(|&dependency| dependency != v)
537            .collect::<Vec<_>>();
538        deps.sort_unstable();
539        deps.dedup();
540        let mut regions = collector
541            .collected_regions
542            .lock()
543            .unwrap()
544            .iter()
545            .copied()
546            .collect::<Vec<_>>();
547        regions.sort_by_key(|region| {
548            let (rows, cols) = region.axis_ranges();
549            (region.sheet_id(), rows.query_bounds(), cols.query_bounds())
550        });
551        regions.dedup();
552        (deps, regions)
553    }
554
555    pub fn get_virtual_deps<R: EvaluationContext>(
556        engine: &Engine<R>,
557        v: VertexId,
558    ) -> Vec<VertexId> {
559        Self::collect(engine, v).0
560    }
561
562    pub(crate) fn get_virtual_regions<R: EvaluationContext>(
563        engine: &Engine<R>,
564        v: VertexId,
565    ) -> Vec<Region> {
566        Self::collect(engine, v).1
567    }
568}
569
570#[cfg(test)]
571mod tests {
572    use super::DynamicRefVirtualDepProvider;
573    use crate::engine::{Engine, EvalConfig};
574    use crate::test_workbook::TestWorkbook;
575
576    /// The query is public and may run before any evaluation request has
577    /// refreshed the per-request function table.
578    #[test]
579    fn virtual_deps_before_any_evaluation_request() {
580        crate::builtins::load_builtins();
581        let mut engine = Engine::new(TestWorkbook::new(), EvalConfig::default());
582        engine.add_sheet("S").expect("sheet");
583        engine
584            .set_cell_formula("S", 1, 1, formualizer_parse::parse("=42").expect("parse"))
585            .expect("A1");
586        engine
587            .set_cell_formula(
588                "S",
589                1,
590                2,
591                formualizer_parse::parse("=INDIRECT(\"A1\")+0").expect("parse"),
592            )
593            .expect("B1");
594        let found: usize = engine
595            .evaluation_vertices()
596            .into_iter()
597            .map(|v| DynamicRefVirtualDepProvider::get_virtual_deps(&engine, v).len())
598            .sum();
599        assert!(
600            found > 0,
601            "INDIRECT's target must be found before the first request"
602        );
603    }
604}
formualizer_eval/engine/virtual_deps.rs

formualizer_eval/engine/
virtual_deps.rs
