velesdb_core/collection/stats/
mod.rs

1//! Collection statistics module for query planning.
2//!
3//! This module provides statistics collection and caching for collections,
4//! enabling cost-based query planning and optimization.
5//!
6//! # EPIC-046 US-001: Collection Statistics
7//!
8//! Implements collection-level statistics including:
9//! - Row count and deleted count
10//! - Column cardinality (distinct values)
11//! - Index statistics (depth, entry count)
12//! - Size metrics (avg row size, total size)
13
14// Reason: Numeric casts in statistics are intentional:
15// - All casts are for computing collection metrics and estimates
16// - f64/usize conversions for cardinality ratios and averages
17// - Values bounded by collection size and column cardinality
18// - Precision loss acceptable for statistics (approximate by design)
19#![allow(clippy::cast_precision_loss)]
20#![allow(clippy::cast_possible_truncation)]
21
22use crate::collection::query_cost::cost_model::OperationCostFactors;
23use serde::{Deserialize, Serialize};
24use std::collections::HashMap;
25
26mod histogram;
27pub(crate) use histogram::next_after;
28pub(crate) use histogram::HistogramBuilder;
29pub use histogram::{Histogram, HistogramBucket};
30
31#[cfg(test)]
32mod tests;
33
34/// Statistics for a collection.
35#[derive(Debug, Clone, Default, Serialize, Deserialize)]
36pub struct CollectionStats {
37    /// Total number of points in the collection.
38    pub total_points: u64,
39    /// Total payload storage footprint in bytes.
40    pub payload_size_bytes: u64,
41    /// Per-field statistics for cost-based planning.
42    pub field_stats: HashMap<String, ColumnStats>,
43    /// Number of active rows
44    pub row_count: u64,
45    /// Number of deleted/tombstoned rows
46    pub deleted_count: u64,
47    /// Average row size in bytes
48    pub avg_row_size_bytes: u64,
49    /// Total collection size in bytes
50    pub total_size_bytes: u64,
51    /// Statistics per column
52    pub column_stats: HashMap<String, ColumnStats>,
53    /// Statistics per index
54    pub index_stats: HashMap<String, IndexStats>,
55    /// Timestamp of last ANALYZE
56    pub last_analyzed_epoch_ms: Option<u64>,
57    /// Calibrated cost factors derived from collection statistics.
58    ///
59    /// `None` if the collection has never been analyzed or stats are invalid.
60    /// Persisted in `collection.stats.json` to survive restarts.
61    #[serde(default, skip_serializing_if = "Option::is_none")]
62    pub calibrated_cost_factors: Option<OperationCostFactors>,
63    /// Graph-shape view of the collection (nodes, edges, labels), filled by
64    /// `ANALYZE` so the MATCH planner and the cost estimator read the same
65    /// source. `None` on stats persisted before 5.2.0 or never analyzed.
66    #[serde(default, skip_serializing_if = "Option::is_none")]
67    pub graph_stats: Option<crate::velesql::match_planner::MatchGraphStats>,
68}
69
70impl CollectionStats {
71    /// Creates empty statistics
72    #[must_use]
73    pub fn new() -> Self {
74        Self::default()
75    }
76
77    /// Creates statistics with basic counts
78    #[must_use]
79    pub fn with_counts(row_count: u64, deleted_count: u64) -> Self {
80        Self {
81            total_points: row_count,
82            row_count,
83            deleted_count,
84            ..Default::default()
85        }
86    }
87
88    /// Returns the live row count (excluding deleted)
89    #[must_use]
90    pub fn live_row_count(&self) -> u64 {
91        self.row_count.saturating_sub(self.deleted_count)
92    }
93
94    /// Returns the deletion ratio (0.0-1.0)
95    #[must_use]
96    pub fn deletion_ratio(&self) -> f64 {
97        if self.row_count == 0 {
98            0.0
99        } else {
100            self.deleted_count as f64 / self.row_count as f64
101        }
102    }
103
104    /// Estimates selectivity for a column based on cardinality
105    #[must_use]
106    pub fn estimate_selectivity(&self, column: &str) -> f64 {
107        if let Some(col_stats) = self.field_stats.get(column) {
108            if col_stats.distinct_values > 0 && self.total_points > 0 {
109                return 1.0 / col_stats.distinct_values as f64;
110            }
111        }
112        if let Some(col_stats) = self.column_stats.get(column) {
113            if col_stats.distinct_count > 0 && self.row_count > 0 {
114                return 1.0 / col_stats.distinct_count as f64;
115            }
116        }
117        // Default: shared structural fallback for an unknown column.
118        selectivity_defaults::EQ
119    }
120
121    /// Returns the histogram for a column, checking both `column_stats` and `field_stats`.
122    ///
123    /// Returns `None` when neither map contains the column or the histogram is
124    /// absent / empty.
125    #[must_use]
126    pub fn get_column_histogram(&self, column: &str) -> Option<&Histogram> {
127        self.column_stats
128            .get(column)
129            .or_else(|| self.field_stats.get(column))
130            .and_then(|cs| cs.histogram.as_ref())
131            .filter(|h| !h.buckets.is_empty())
132    }
133
134    /// Sets the last analyzed timestamp to now
135    pub fn mark_analyzed(&mut self) {
136        self.last_analyzed_epoch_ms = Some(
137            std::time::SystemTime::now()
138                .duration_since(std::time::UNIX_EPOCH)
139                .map_or(0, |d| d.as_millis() as u64),
140        );
141    }
142}
143
144/// Shared structural fallback selectivities.
145///
146/// Single source for the constants used when no histogram or cardinality
147/// data can answer — consumed by both the plan-time AST estimator
148/// (`velesql::cost_estimator`) and the runtime filter estimator below, so
149/// the two paths cannot drift apart silently.
150pub(crate) mod selectivity_defaults {
151    /// Equality / IS NULL with no statistics.
152    pub(crate) const EQ: f64 = 0.1;
153    /// Range, pattern and containment predicates with no statistics.
154    #[cfg(feature = "persistence")]
155    pub(crate) const RANGE: f64 = 0.3;
156    /// Per-value contribution of an IN list.
157    #[cfg(feature = "persistence")]
158    pub(crate) const IN_PER_VALUE: f64 = 0.05;
159    /// Cap on an IN list's total selectivity.
160    #[cfg(feature = "persistence")]
161    pub(crate) const IN_CAP: f64 = 0.8;
162    /// Negative predicates (`!=`, IS NOT NULL).
163    #[cfg(feature = "persistence")]
164    pub(crate) const NEGATION: f64 = 0.9;
165    /// Floor applied under conjunction/negation so an over-confident
166    /// estimate never predicts zero rows.
167    pub(crate) const FLOOR: f64 = 0.01;
168}
169
170#[cfg(feature = "persistence")]
171impl CollectionStats {
172    /// Estimates the fraction of rows matching a runtime metadata filter.
173    ///
174    /// Histogram-backed where `ANALYZE` has produced one for the field,
175    /// cardinality-backed otherwise, and falling back to the same
176    /// structural constants as the plan-time estimator. Runtime mirror of
177    /// `CostEstimator::estimate_condition_selectivity`, which speaks the
178    /// AST `Condition` — this one speaks `crate::filter::Condition`.
179    #[must_use]
180    pub(crate) fn estimate_runtime_filter_selectivity(&self, filter: &crate::Filter) -> f64 {
181        self.estimate_runtime_condition_selectivity(&filter.condition)
182    }
183
184    pub(crate) fn estimate_runtime_condition_selectivity(
185        &self,
186        cond: &crate::filter::Condition,
187    ) -> f64 {
188        use crate::filter::Condition as C;
189        use selectivity_defaults as d;
190        match cond {
191            C::Eq { field, value } => self.runtime_eq_selectivity(field, value),
192            C::Neq { field, value } => {
193                (1.0 - self.runtime_eq_selectivity(field, value)).clamp(d::FLOOR, 1.0)
194            }
195            C::Gt { field, value } => self.runtime_lt_complement(field, value, true),
196            C::Gte { field, value } => self.runtime_lt_complement(field, value, false),
197            C::Lt { field, value } => self.runtime_lt_selectivity(field, value, false),
198            C::Lte { field, value } => self.runtime_lt_selectivity(field, value, true),
199            C::In { field, values } => {
200                let sum: f64 = values
201                    .iter()
202                    .map(|v| self.runtime_eq_selectivity(field, v))
203                    .sum();
204                sum.min(d::IN_CAP)
205            }
206            C::IsNull { field } => self.runtime_null_ratio(field),
207            C::IsNotNull { field } => (1.0 - self.runtime_null_ratio(field)).clamp(d::FLOOR, 1.0),
208            C::And { conditions } => conditions
209                .iter()
210                .map(|c| self.estimate_runtime_condition_selectivity(c))
211                .product::<f64>()
212                .max(d::FLOOR),
213            C::Or { conditions } => conditions
214                .iter()
215                .map(|c| self.estimate_runtime_condition_selectivity(c))
216                .sum::<f64>()
217                .min(1.0),
218            C::Not { condition } => {
219                (1.0 - self.estimate_runtime_condition_selectivity(condition)).max(d::FLOOR)
220            }
221            C::Contains { .. }
222            | C::Like { .. }
223            | C::ILike { .. }
224            | C::ArrayContains { .. }
225            | C::ArrayContainsAny { .. }
226            | C::ArrayContainsAll { .. }
227            | C::GeoDistance { .. }
228            | C::GeoBbox { .. } => d::RANGE,
229        }
230    }
231
232    /// Histogram equality estimate, then cardinality, then the shared default.
233    fn runtime_eq_selectivity(&self, field: &str, value: &serde_json::Value) -> f64 {
234        if let (Some(hist), Some(v)) = (self.get_column_histogram(field), value.as_f64()) {
235            return hist.estimate_eq_selectivity(v).clamp(0.0, 1.0);
236        }
237        self.estimate_selectivity(field)
238    }
239
240    /// Histogram `<` / `<=` estimate; `RANGE` fallback without one.
241    fn runtime_lt_selectivity(
242        &self,
243        field: &str,
244        value: &serde_json::Value,
245        inclusive: bool,
246    ) -> f64 {
247        if let (Some(hist), Some(v)) = (self.get_column_histogram(field), value.as_f64()) {
248            let bound = if inclusive { next_after(v) } else { v };
249            return hist.estimate_lt_selectivity(bound).clamp(0.0, 1.0);
250        }
251        selectivity_defaults::RANGE
252    }
253
254    /// Histogram `>` / `>=` as the complement of `<=` / `<`.
255    fn runtime_lt_complement(&self, field: &str, value: &serde_json::Value, strict: bool) -> f64 {
256        if let (Some(hist), Some(v)) = (self.get_column_histogram(field), value.as_f64()) {
257            let bound = if strict { next_after(v) } else { v };
258            return (1.0 - hist.estimate_lt_selectivity(bound)).clamp(0.0, 1.0);
259        }
260        selectivity_defaults::RANGE
261    }
262
263    /// Observed null ratio for a field; `EQ` default when never analyzed.
264    fn runtime_null_ratio(&self, field: &str) -> f64 {
265        self.field_stats
266            .get(field)
267            .or_else(|| self.column_stats.get(field))
268            .map_or(selectivity_defaults::EQ, |s| {
269                #[allow(clippy::cast_precision_loss)]
270                let ratio = s.null_count as f64 / self.total_points.max(1) as f64;
271                ratio.clamp(0.0, 1.0)
272            })
273    }
274}
275
276/// Statistics for a single column.
277#[derive(Debug, Clone, Default, Serialize, Deserialize)]
278pub struct ColumnStats {
279    /// Column name
280    pub name: String,
281    /// Number of null values
282    pub null_count: u64,
283    /// Number of distinct values (cardinality)
284    pub distinct_count: u64,
285    /// Number of distinct values (CBO alias).
286    pub distinct_values: u64,
287    /// Minimum value (serialized)
288    pub min_value: Option<String>,
289    /// Maximum value (serialized)
290    pub max_value: Option<String>,
291    /// Average value size in bytes
292    pub avg_size_bytes: u64,
293    /// Optional histogram for selectivity estimates.
294    pub histogram: Option<Histogram>,
295}
296
297impl ColumnStats {
298    /// Creates new column stats
299    #[must_use]
300    pub fn new(name: impl Into<String>) -> Self {
301        Self {
302            name: name.into(),
303            ..Default::default()
304        }
305    }
306
307    /// Sets cardinality
308    #[must_use]
309    pub fn with_distinct_count(mut self, count: u64) -> Self {
310        self.distinct_count = count;
311        self.distinct_values = count;
312        self
313    }
314
315    /// Sets null count
316    #[must_use]
317    pub fn with_null_count(mut self, count: u64) -> Self {
318        self.null_count = count;
319        self
320    }
321}
322
323/// Statistics for an index.
324#[derive(Debug, Clone, Default, Serialize, Deserialize)]
325pub struct IndexStats {
326    /// Index name
327    pub name: String,
328    /// Index type (HNSW, PropertyIndex, etc.)
329    pub index_type: String,
330    /// Number of entries in the index
331    pub entry_count: u64,
332    /// Index depth (for tree-based indexes)
333    pub depth: u32,
334    /// Index size in bytes
335    pub size_bytes: u64,
336}
337
338impl IndexStats {
339    /// Creates new index stats
340    #[must_use]
341    pub fn new(name: impl Into<String>, index_type: impl Into<String>) -> Self {
342        Self {
343            name: name.into(),
344            index_type: index_type.into(),
345            ..Default::default()
346        }
347    }
348
349    /// Sets entry count
350    #[must_use]
351    pub fn with_entry_count(mut self, count: u64) -> Self {
352        self.entry_count = count;
353        self
354    }
355
356    /// Sets depth
357    #[must_use]
358    pub fn with_depth(mut self, depth: u32) -> Self {
359        self.depth = depth;
360        self
361    }
362}
363
364/// Statistics collector for building CollectionStats.
365#[derive(Debug, Default)]
366pub struct StatsCollector {
367    stats: CollectionStats,
368}
369
370impl StatsCollector {
371    /// Creates a new collector
372    #[must_use]
373    pub fn new() -> Self {
374        Self::default()
375    }
376
377    /// Sets row count
378    pub fn set_row_count(&mut self, count: u64) {
379        self.stats.row_count = count;
380        self.stats.total_points = count;
381    }
382
383    /// Sets deleted count
384    pub fn set_deleted_count(&mut self, count: u64) {
385        self.stats.deleted_count = count;
386    }
387
388    /// Sets total size
389    pub fn set_total_size(&mut self, size: u64) {
390        self.stats.total_size_bytes = size;
391        self.stats.payload_size_bytes = size;
392    }
393
394    /// Adds column statistics
395    pub fn add_column_stats(&mut self, stats: ColumnStats) {
396        self.stats
397            .column_stats
398            .insert(stats.name.clone(), stats.clone());
399        self.stats.field_stats.insert(stats.name.clone(), stats);
400    }
401
402    /// Adds index statistics
403    pub fn add_index_stats(&mut self, stats: IndexStats) {
404        self.stats.index_stats.insert(stats.name.clone(), stats);
405    }
406
407    /// Builds a histogram for a column from sampled values and stores it.
408    ///
409    /// Called by `Collection::analyze()` for each Int, Float, and String column.
410    /// Uses `HistogramBuilder` to construct an equi-depth histogram, then attaches
411    /// it to the corresponding `ColumnStats` entry (creating one if absent).
412    pub fn build_histogram(&mut self, column_name: &str, values: &mut [f64], num_buckets: usize) {
413        let histogram = HistogramBuilder::new(num_buckets).build(values);
414        self.stats
415            .column_stats
416            .entry(column_name.to_owned())
417            .or_insert_with(|| ColumnStats::new(column_name))
418            .histogram = Some(histogram.clone());
419        self.stats
420            .field_stats
421            .entry(column_name.to_owned())
422            .or_insert_with(|| ColumnStats::new(column_name))
423            .histogram = Some(histogram);
424    }
425
426    /// Builds the final CollectionStats
427    #[must_use]
428    pub fn build(mut self) -> CollectionStats {
429        // Calculate average row size
430        if let Some(avg) = self
431            .stats
432            .total_size_bytes
433            .checked_div(self.stats.row_count)
434        {
435            self.stats.avg_row_size_bytes = avg;
436        }
437
438        self.stats.mark_analyzed();
439        self.stats
440    }
441}
velesdb_core/collection/stats/mod.rs

velesdb_core/collection/stats/
mod.rs
