formualizer_eval/engine/
resource_observability.rs

1use super::{
2    DiskScratchPolicy, EvaluationIncompleteReason, FormulaPlaneMode, ResourceLedgerSnapshot,
3};
4use formualizer_common::ResourceExhaustionReason;
5
6/// Stable classification for evaluation limits. C0 is observational only: these classes do not
7/// alter limit enforcement or fallback selection.
8#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
9#[non_exhaustive]
10pub enum EvaluationResourceClass {
11    SemanticFormat,
12    Admission,
13    RetainedMemory,
14    ScratchMemory,
15    WorkTime,
16    Optimization,
17}
18
19/// Stable reason vocabulary for observed evaluation-resource boundaries.
20#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
21#[non_exhaustive]
22pub enum EvaluationResourceReason {
23    FormulaPlaneTopologyCandidates,
24    FormulaPlaneTopologyEdges,
25    FormulaPlaneTopologyRetainedBytes,
26    FormulaPlaneMaterializationCells,
27    FormulaReplayEncodedBytes,
28    FormulaReplayMemoryBytes,
29    FormulaReplayDiskBytes,
30    FormulaReplayFiles,
31    EvaluationCancelled,
32    EvaluationError,
33}
34
35impl EvaluationResourceClass {
36    pub const fn as_str(self) -> &'static str {
37        match self {
38            Self::SemanticFormat => "semantic_format",
39            Self::Admission => "admission",
40            Self::RetainedMemory => "retained_memory",
41            Self::ScratchMemory => "scratch_memory",
42            Self::WorkTime => "work_time",
43            Self::Optimization => "optimization",
44        }
45    }
46}
47
48impl EvaluationResourceReason {
49    pub const fn class(self) -> EvaluationResourceClass {
50        match self {
51            Self::FormulaPlaneTopologyCandidates | Self::FormulaPlaneTopologyEdges => {
52                EvaluationResourceClass::Optimization
53            }
54            Self::FormulaPlaneTopologyRetainedBytes => EvaluationResourceClass::RetainedMemory,
55            Self::FormulaPlaneMaterializationCells
56            | Self::FormulaReplayEncodedBytes
57            | Self::FormulaReplayDiskBytes
58            | Self::FormulaReplayFiles => EvaluationResourceClass::Admission,
59            Self::FormulaReplayMemoryBytes => EvaluationResourceClass::ScratchMemory,
60            Self::EvaluationCancelled | Self::EvaluationError => EvaluationResourceClass::WorkTime,
61        }
62    }
63
64    pub const fn as_str(self) -> &'static str {
65        match self {
66            Self::FormulaPlaneTopologyCandidates => "formula_plane_topology_candidates",
67            Self::FormulaPlaneTopologyEdges => "formula_plane_topology_edges",
68            Self::FormulaPlaneTopologyRetainedBytes => "formula_plane_topology_retained_bytes",
69            Self::FormulaPlaneMaterializationCells => "formula_plane_materialization_cells",
70            Self::FormulaReplayEncodedBytes => "formula_replay_encoded_bytes",
71            Self::FormulaReplayMemoryBytes => "formula_replay_memory_bytes",
72            Self::FormulaReplayDiskBytes => "formula_replay_disk_bytes",
73            Self::FormulaReplayFiles => "formula_replay_files",
74            Self::EvaluationCancelled => "evaluation_cancelled",
75            Self::EvaluationError => "evaluation_error",
76        }
77    }
78}
79
80#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
81#[non_exhaustive]
82pub enum EvaluationRequestKind {
83    Vertex,
84    Targeted,
85    TargetPreparation,
86    RecalcPlan,
87    #[default]
88    Full,
89    FullWithDelta,
90    Cell,
91    Cells,
92    CellsCancellable,
93    CellsWithDelta,
94    FullCancellable,
95    TargetedCancellable,
96    FullLogged,
97}
98
99#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
100#[non_exhaustive]
101pub enum EvaluationRequestOutcome {
102    #[default]
103    InProgress,
104    Success,
105    Cancelled,
106    Error,
107}
108
109#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
110#[non_exhaustive]
111pub enum FormulaPlaneTopologyStrategy {
112    #[default]
113    NotUsed,
114    Legacy,
115    SkippedNoActiveSpans,
116    SkippedNoDirtyWork,
117    Cached,
118    CompiledAndCached,
119    ExactPagedIndexed,
120    ExactInMemoryRuns,
121    ExactNativeScratch,
122    ExactRepeatedPasses,
123    CapacityFallbackMaterialization,
124}
125
126#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
127#[non_exhaustive]
128pub enum FormulaPlaneTopologyCacheOutcome {
129    #[default]
130    NotUsed,
131    Hit,
132    Built,
133    SkippedOverflow,
134    SkippedDynamicLegacy,
135}
136
137#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
138#[non_exhaustive]
139pub enum FormulaDirtyLeaseOutcome {
140    #[default]
141    NotAcquired,
142    Acquired,
143    Empty,
144    Acknowledged,
145    AcknowledgedPartial,
146    AcknowledgedEmpty,
147    RetainedOnCancellation,
148    RetainedOnError,
149}
150
151/// Bounded routing telemetry for legacy-island contraction. The fixed-size
152/// representation prevents observability from introducing an unaccounted,
153/// input-sized allocation under retained-memory pressure.
154#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
155#[non_exhaustive]
156pub enum FormulaPlaneRoute {
157    #[default]
158    GlobalMixed,
159    ContractedLegacyIsland,
160}
161
162#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
163#[non_exhaustive]
164pub enum FormulaPlaneRoutePhase {
165    #[default]
166    Planned,
167    Executed,
168}
169
170#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
171#[non_exhaustive]
172pub enum FormulaPlaneRouteTransitionReason {
173    #[default]
174    ProvenIsolated,
175    DynamicReference,
176    NamedDependency,
177    SpillOrArray,
178    StructuralSummaryUncertain,
179    UnsupportedReadSummary,
180    BoundaryDiscoveryOverflow,
181    SpanCycleDemotion,
182    RuntimeReplan,
183}
184
185pub const FORMULA_PLANE_ROUTE_EVENT_CAPACITY: usize = 16;
186pub const FORMULA_PLANE_ROUTE_EVENT_SHEET_CAPACITY: usize = 8;
187
188#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
189#[non_exhaustive]
190pub struct FormulaPlaneRouteEvent {
191    /// Opaque request/session-local correlation identity derived from island membership; not
192    /// a durable identifier.
193    pub island_id: u64,
194    pub phase: FormulaPlaneRoutePhase,
195    pub route: FormulaPlaneRoute,
196    /// Opaque request/session-local sheet correlation IDs, sorted and truncated at the fixed
197    /// capacity; not durable identifiers.
198    pub sheet_ids: [u16; FORMULA_PLANE_ROUTE_EVENT_SHEET_CAPACITY],
199    pub sheet_count: u8,
200    /// Opaque request/session-local correlation epoch, not a durable identifier.
201    pub authority_epoch: u64,
202    /// Opaque request/session-local correlation epoch, not a durable identifier.
203    pub index_epoch: u64,
204    pub transition_reason: FormulaPlaneRouteTransitionReason,
205    pub demotion_generation: u32,
206    pub replan_generation: u32,
207}
208
209impl EvaluationRequestKind {
210    pub const fn as_str(self) -> &'static str {
211        match self {
212            Self::Vertex => "vertex",
213            Self::Targeted => "targeted",
214            Self::TargetPreparation => "target_preparation",
215            Self::RecalcPlan => "recalc_plan",
216            Self::Full => "full",
217            Self::FullWithDelta => "full_with_delta",
218            Self::Cell => "cell",
219            Self::Cells => "cells",
220            Self::CellsCancellable => "cells_cancellable",
221            Self::CellsWithDelta => "cells_with_delta",
222            Self::FullCancellable => "full_cancellable",
223            Self::TargetedCancellable => "targeted_cancellable",
224            Self::FullLogged => "full_logged",
225        }
226    }
227}
228
229impl EvaluationRequestOutcome {
230    pub const fn as_str(self) -> &'static str {
231        match self {
232            Self::InProgress => "in_progress",
233            Self::Success => "success",
234            Self::Cancelled => "cancelled",
235            Self::Error => "error",
236        }
237    }
238}
239
240impl FormulaPlaneTopologyStrategy {
241    pub const fn as_str(self) -> &'static str {
242        match self {
243            Self::NotUsed => "not_used",
244            Self::Legacy => "legacy",
245            Self::SkippedNoActiveSpans => "skipped_no_active_spans",
246            Self::SkippedNoDirtyWork => "skipped_no_dirty_work",
247            Self::Cached => "cached",
248            Self::CompiledAndCached => "compiled_and_cached",
249            Self::ExactPagedIndexed => "exact_paged_indexed",
250            Self::ExactInMemoryRuns => "exact_in_memory_runs",
251            Self::ExactNativeScratch => "exact_native_scratch",
252            Self::ExactRepeatedPasses => "exact_repeated_passes",
253            Self::CapacityFallbackMaterialization => "capacity_fallback_materialization",
254        }
255    }
256
257    pub(crate) const fn severity(self) -> u8 {
258        match self {
259            Self::NotUsed => 0,
260            Self::Legacy => 1,
261            Self::SkippedNoActiveSpans => 2,
262            Self::SkippedNoDirtyWork => 3,
263            Self::Cached => 4,
264            Self::CompiledAndCached => 5,
265            Self::ExactPagedIndexed => 6,
266            Self::ExactInMemoryRuns => 7,
267            Self::ExactNativeScratch => 8,
268            Self::ExactRepeatedPasses => 9,
269            Self::CapacityFallbackMaterialization => 10,
270        }
271    }
272}
273
274impl FormulaPlaneTopologyCacheOutcome {
275    pub const fn as_str(self) -> &'static str {
276        match self {
277            Self::NotUsed => "not_used",
278            Self::Hit => "hit",
279            Self::Built => "built",
280            Self::SkippedOverflow => "skipped_overflow",
281            Self::SkippedDynamicLegacy => "skipped_dynamic_legacy",
282        }
283    }
284
285    pub(crate) const fn severity(self) -> u8 {
286        match self {
287            Self::NotUsed => 0,
288            Self::Hit => 1,
289            Self::Built => 2,
290            Self::SkippedOverflow => 3,
291            Self::SkippedDynamicLegacy => 4,
292        }
293    }
294}
295
296impl FormulaDirtyLeaseOutcome {
297    pub const fn as_str(self) -> &'static str {
298        match self {
299            Self::NotAcquired => "not_acquired",
300            Self::Acquired => "acquired",
301            Self::Empty => "empty",
302            Self::Acknowledged => "acknowledged",
303            Self::AcknowledgedPartial => "acknowledged_partial",
304            Self::AcknowledgedEmpty => "acknowledged_empty",
305            Self::RetainedOnCancellation => "retained_on_cancellation",
306            Self::RetainedOnError => "retained_on_error",
307        }
308    }
309}
310
311#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
312#[non_exhaustive]
313pub struct FormulaPlaneTopologyRequestStats {
314    /// Most severe strategy observed during the request.
315    pub strategy: FormulaPlaneTopologyStrategy,
316    /// Most severe cache outcome observed during the request.
317    pub cache_outcome: FormulaPlaneTopologyCacheOutcome,
318    pub cache_hit_events: u64,
319    pub cache_build_events: u64,
320    pub cache_skip_events: u64,
321    pub cache_skip_streak: u64,
322    pub exact_pass_count: u64,
323    /// Delete-on-drop native topology edge-record bytes written by this request.
324    pub native_topology_disk_bytes: u64,
325    pub candidate_cap: Option<u64>,
326    pub edge_cap: Option<u64>,
327    pub retained_byte_cap: Option<u64>,
328    pub operator_guidance: Option<&'static str>,
329    /// Sum across topology build attempts; cache hits do not replay prior build work.
330    pub producers_observed: u64,
331    pub candidates_observed: u64,
332    pub edges_observed: u64,
333    pub retained_bytes_observed: u64,
334    pub candidate_cap_hits: u64,
335    pub edge_cap_hits: u64,
336    pub byte_cap_hits: u64,
337    pub overflow_reason: Option<EvaluationResourceReason>,
338    pub incomplete_reason: Option<EvaluationIncompleteReason>,
339    /// Bounded per-route records. Entries beyond the fixed capacity are
340    /// counted rather than allocated.
341    pub route_events: [FormulaPlaneRouteEvent; FORMULA_PLANE_ROUTE_EVENT_CAPACITY],
342    pub route_event_count: u8,
343    pub route_events_dropped: u64,
344    /// Inline telemetry bytes plus membership bytes charged to the retained
345    /// mixed-cache ledger.
346    pub route_event_bytes_observed: u64,
347    pub island_membership_vertices: u64,
348    pub island_membership_retained_bytes: u64,
349    pub legacy_relationships_omitted: u64,
350    pub boundary_relationships_retained: u64,
351}
352
353#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
354#[non_exhaustive]
355pub struct EvaluationResourceLedgerRequestStats {
356    pub retained_limit: Option<u64>,
357    pub mixed_cache_limit: Option<u64>,
358    pub retained_current: u64,
359    pub retained_peak: u64,
360    pub scratch_limit: Option<u64>,
361    pub schedule_discovery_limit: Option<u64>,
362    pub scratch_current: u64,
363    pub scratch_peak: u64,
364    pub disk_scratch_policy: Option<DiskScratchPolicy>,
365    pub work_limit: Option<u64>,
366    pub work_charged: u64,
367    pub deadline_ns: Option<u64>,
368    pub deadline_checkpoints: u64,
369    pub exhaustion: Option<ResourceExhaustionReason>,
370}
371
372impl EvaluationResourceLedgerRequestStats {
373    pub(crate) fn update(&mut self, snapshot: ResourceLedgerSnapshot) {
374        self.retained_limit = snapshot.retained_limit;
375        self.mixed_cache_limit = snapshot.mixed_cache_limit;
376        self.retained_current = snapshot.retained_current;
377        self.retained_peak = snapshot.retained_peak;
378        self.scratch_limit = snapshot.scratch_limit;
379        self.schedule_discovery_limit = snapshot.schedule_discovery_limit;
380        self.scratch_current = snapshot.scratch_current;
381        self.scratch_peak = snapshot.scratch_peak;
382        self.disk_scratch_policy = snapshot.disk_scratch_policy;
383        self.work_limit = snapshot.work_limit;
384        self.work_charged = snapshot.work_charged;
385        self.deadline_ns = snapshot.deadline_ns;
386        self.deadline_checkpoints = snapshot.deadline_checkpoints;
387        self.exhaustion = snapshot.exhaustion;
388    }
389}
390
391#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
392#[non_exhaustive]
393pub struct EvaluationRequestPhaseTimings {
394    pub total_ns: u64,
395    pub staged_prepare_ns: u64,
396    pub topology_ns: u64,
397    pub materialization_ns: u64,
398    pub evaluation_ns: u64,
399}
400
401#[derive(Debug, Clone, Copy, PartialEq, Eq)]
402#[non_exhaustive]
403pub struct EvaluationResourceRequestStats {
404    pub request_id: u64,
405    pub kind: EvaluationRequestKind,
406    pub formula_plane_mode: FormulaPlaneMode,
407    pub outcome: EvaluationRequestOutcome,
408    pub staged_selected: u64,
409    pub staged_retained: u64,
410    pub target_requested: u64,
411    pub target_normalized_regions: u64,
412    /// 0 = not used/exact, 1 = sheets, 2 = workbook.
413    pub target_scope_level: u8,
414    /// Stable bit set keyed by `OpaqueReason` discriminants.
415    pub target_widening_reason_bits: u64,
416    pub graph_source_scratch_estimated: u64,
417    pub graph_source_scratch_observed: u64,
418    pub target_commit_estimated_work: u64,
419    pub target_commit_actual_work: u64,
420    pub target_commit_window_ns: u64,
421    pub target_admission_failure: Option<ResourceExhaustionReason>,
422    pub evaluation_commit_preflight_count: u64,
423    pub evaluation_commit_estimated_ns: u64,
424    pub evaluation_commit_actual_ns: u64,
425    pub runtime_replan_rounds: u64,
426    pub runtime_widening_rounds: u64,
427    pub workbook_exact_attempts: u64,
428    pub topology: FormulaPlaneTopologyRequestStats,
429    pub fallback_materialized_cells: u64,
430    pub cycle_materialized_cells: u64,
431    pub dirty_lease: FormulaDirtyLeaseOutcome,
432    pub ledger: EvaluationResourceLedgerRequestStats,
433    pub phases: EvaluationRequestPhaseTimings,
434}
435
436impl EvaluationResourceRequestStats {
437    pub(crate) fn new(
438        request_id: u64,
439        kind: EvaluationRequestKind,
440        formula_plane_mode: FormulaPlaneMode,
441        staged_retained: usize,
442    ) -> Self {
443        Self {
444            request_id,
445            kind,
446            formula_plane_mode,
447            outcome: EvaluationRequestOutcome::InProgress,
448            staged_selected: 0,
449            staged_retained: staged_retained as u64,
450            target_requested: 0,
451            target_normalized_regions: 0,
452            target_scope_level: 0,
453            target_widening_reason_bits: 0,
454            graph_source_scratch_estimated: 0,
455            graph_source_scratch_observed: 0,
456            target_commit_estimated_work: 0,
457            target_commit_actual_work: 0,
458            target_commit_window_ns: 0,
459            target_admission_failure: None,
460            evaluation_commit_preflight_count: 0,
461            evaluation_commit_estimated_ns: 0,
462            evaluation_commit_actual_ns: 0,
463            runtime_replan_rounds: 0,
464            runtime_widening_rounds: 0,
465            workbook_exact_attempts: 0,
466            topology: FormulaPlaneTopologyRequestStats {
467                strategy: if formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
468                    FormulaPlaneTopologyStrategy::NotUsed
469                } else {
470                    FormulaPlaneTopologyStrategy::Legacy
471                },
472                ..FormulaPlaneTopologyRequestStats::default()
473            },
474            fallback_materialized_cells: 0,
475            cycle_materialized_cells: 0,
476            dirty_lease: FormulaDirtyLeaseOutcome::NotAcquired,
477            ledger: EvaluationResourceLedgerRequestStats::default(),
478            phases: EvaluationRequestPhaseTimings::default(),
479        }
480    }
481}
482
483/// Cumulative observational counters since engine creation or the last explicit telemetry reset.
484/// Resetting these counters never resets the monotonic request ID sequence.
485#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
486#[non_exhaustive]
487pub struct EvaluationResourceBaselineStats {
488    pub last_request_id: u64,
489    pub requests_started: u64,
490    pub requests_succeeded: u64,
491    pub requests_cancelled: u64,
492    pub requests_errored: u64,
493    pub staged_selected_total: u64,
494    pub topology_cache_hits: u64,
495    pub topology_cache_builds: u64,
496    pub topology_cache_skips: u64,
497    pub topology_cache_skip_streak_current: u64,
498    pub topology_cache_skip_streak_max: u64,
499    pub topology_exact_passes_total: u64,
500    pub topology_native_disk_bytes_total: u64,
501    pub topology_candidate_cap_hits: u64,
502    pub topology_edge_cap_hits: u64,
503    pub topology_byte_cap_hits: u64,
504    pub topology_candidates_observed_total: u64,
505    pub topology_edges_observed_total: u64,
506    pub topology_retained_bytes_observed_max: u64,
507    pub fallback_materialized_cells_total: u64,
508    pub cycle_materialized_cells_total: u64,
509    pub dirty_leases_acknowledged: u64,
510    pub dirty_leases_retained_on_cancel: u64,
511    pub dirty_leases_retained_on_error: u64,
512    pub ledger_retained_peak: u64,
513    pub ledger_scratch_peak: u64,
514    pub ledger_work_charged_total: u64,
515    pub ledger_deadline_checkpoints: u64,
516    pub ledger_exhaustions: u64,
517    pub last_ledger_exhaustion: Option<ResourceExhaustionReason>,
518    pub total_request_ns: u64,
519    pub staged_prepare_ns: u64,
520    pub topology_ns: u64,
521    pub materialization_ns: u64,
522    pub evaluation_ns: u64,
523}
524
525impl EvaluationResourceBaselineStats {
526    pub(crate) fn record_started(&mut self, request_id: u64) {
527        self.last_request_id = request_id;
528        self.requests_started = self.requests_started.saturating_add(1);
529    }
530
531    pub(crate) fn record_finished(&mut self, stats: &EvaluationResourceRequestStats) {
532        match stats.outcome {
533            EvaluationRequestOutcome::Success => {
534                self.requests_succeeded = self.requests_succeeded.saturating_add(1)
535            }
536            EvaluationRequestOutcome::Cancelled => {
537                self.requests_cancelled = self.requests_cancelled.saturating_add(1)
538            }
539            EvaluationRequestOutcome::Error => {
540                self.requests_errored = self.requests_errored.saturating_add(1)
541            }
542            EvaluationRequestOutcome::InProgress => {}
543        }
544        self.staged_selected_total = self
545            .staged_selected_total
546            .saturating_add(stats.staged_selected);
547        self.topology_cache_hits = self
548            .topology_cache_hits
549            .saturating_add(stats.topology.cache_hit_events);
550        self.topology_cache_builds = self
551            .topology_cache_builds
552            .saturating_add(stats.topology.cache_build_events);
553        self.topology_cache_skips = self
554            .topology_cache_skips
555            .saturating_add(stats.topology.cache_skip_events);
556        self.topology_cache_skip_streak_current = stats.topology.cache_skip_streak;
557        self.topology_cache_skip_streak_max = self
558            .topology_cache_skip_streak_max
559            .max(stats.topology.cache_skip_streak);
560        self.topology_exact_passes_total = self
561            .topology_exact_passes_total
562            .saturating_add(stats.topology.exact_pass_count);
563        self.topology_native_disk_bytes_total = self
564            .topology_native_disk_bytes_total
565            .saturating_add(stats.topology.native_topology_disk_bytes);
566        self.topology_candidate_cap_hits = self
567            .topology_candidate_cap_hits
568            .saturating_add(stats.topology.candidate_cap_hits);
569        self.topology_edge_cap_hits = self
570            .topology_edge_cap_hits
571            .saturating_add(stats.topology.edge_cap_hits);
572        self.topology_byte_cap_hits = self
573            .topology_byte_cap_hits
574            .saturating_add(stats.topology.byte_cap_hits);
575        self.topology_candidates_observed_total = self
576            .topology_candidates_observed_total
577            .saturating_add(stats.topology.candidates_observed);
578        self.topology_edges_observed_total = self
579            .topology_edges_observed_total
580            .saturating_add(stats.topology.edges_observed);
581        self.topology_retained_bytes_observed_max = self
582            .topology_retained_bytes_observed_max
583            .max(stats.topology.retained_bytes_observed);
584        self.fallback_materialized_cells_total = self
585            .fallback_materialized_cells_total
586            .saturating_add(stats.fallback_materialized_cells);
587        self.cycle_materialized_cells_total = self
588            .cycle_materialized_cells_total
589            .saturating_add(stats.cycle_materialized_cells);
590        match stats.dirty_lease {
591            FormulaDirtyLeaseOutcome::Acknowledged
592            | FormulaDirtyLeaseOutcome::AcknowledgedPartial
593            | FormulaDirtyLeaseOutcome::AcknowledgedEmpty => {
594                self.dirty_leases_acknowledged = self.dirty_leases_acknowledged.saturating_add(1)
595            }
596            FormulaDirtyLeaseOutcome::RetainedOnCancellation => {
597                self.dirty_leases_retained_on_cancel =
598                    self.dirty_leases_retained_on_cancel.saturating_add(1)
599            }
600            FormulaDirtyLeaseOutcome::RetainedOnError => {
601                self.dirty_leases_retained_on_error =
602                    self.dirty_leases_retained_on_error.saturating_add(1)
603            }
604            _ => {}
605        }
606        self.ledger_retained_peak = self.ledger_retained_peak.max(stats.ledger.retained_peak);
607        self.ledger_scratch_peak = self.ledger_scratch_peak.max(stats.ledger.scratch_peak);
608        self.ledger_work_charged_total = self
609            .ledger_work_charged_total
610            .saturating_add(stats.ledger.work_charged);
611        self.ledger_deadline_checkpoints = self
612            .ledger_deadline_checkpoints
613            .saturating_add(stats.ledger.deadline_checkpoints);
614        if let Some(reason) = stats.ledger.exhaustion {
615            self.ledger_exhaustions = self.ledger_exhaustions.saturating_add(1);
616            self.last_ledger_exhaustion = Some(reason);
617        }
618        self.total_request_ns = self.total_request_ns.saturating_add(stats.phases.total_ns);
619        self.staged_prepare_ns = self
620            .staged_prepare_ns
621            .saturating_add(stats.phases.staged_prepare_ns);
622        self.topology_ns = self.topology_ns.saturating_add(stats.phases.topology_ns);
623        self.materialization_ns = self
624            .materialization_ns
625            .saturating_add(stats.phases.materialization_ns);
626        self.evaluation_ns = self
627            .evaluation_ns
628            .saturating_add(stats.phases.evaluation_ns);
629    }
630}
formualizer_eval/engine/resource_observability.rs

formualizer_eval/engine/
resource_observability.rs
