Skip to main content

uqa_engine/
statistics.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Durable automatic-statistics scheduling, independent of query planning.
8
9mod cache;
10mod coordinator;
11pub(crate) use coordinator::{shared_statistics, StatisticsCoordinator};
12pub(crate) mod value_size;
13mod worker;
14pub(crate) use cache::StatisticsSnapshots;
15
16use std::collections::BTreeMap;
17use std::sync::atomic::{AtomicUsize, Ordering};
18use std::sync::{mpsc, Arc};
19use std::time::{SystemTime, UNIX_EPOCH};
20
21use parking_lot::Mutex;
22use uqa_storage::StorageBackendResult;
23
24use crate::{Engine, TableState};
25
26pub(crate) use uqa_storage::statistics_maintenance::StatisticsMaintenance as MaintenanceState;
27
28#[derive(Clone, Default)]
29pub(crate) struct StatisticsChange {
30    object_id: [u8; 16],
31    count: u64,
32    reset: bool,
33    /// For changes a session keeps: a commit sequence at or after the last commit that made them, or `None` when its provider numbers no commits. An analysis that sampled at or after it has seen them.
34    through: Option<u64>,
35}
36
37pub(crate) type StatisticsChanges = BTreeMap<String, StatisticsChange>;
38
39/// What a commit did with the statistics changes of its transaction. The session takes it over once the commit has succeeded: a failed or undone commit leaves what the session kept as it was.
40#[derive(Clone, Default)]
41pub(crate) struct StatisticsSettlement {
42    /// Tables whose maintenance record the transaction wrote, with every change the session had kept of them.
43    recorded: Vec<String>,
44    /// Changes the session keeps to itself, including the ones it kept before.
45    kept: StatisticsChanges,
46}
47
48impl StatisticsSettlement {
49    /// Whether the transaction wrote a maintenance record, which the automatic statistics worker should see.
50    pub(crate) fn recorded(&self) -> bool {
51        !self.recorded.is_empty()
52    }
53}
54
55/// Process-local health of the database's automatic statistics worker.
56/// A failed refresh retains its durable pending state and is retried.
57#[derive(Clone, Debug, Default)]
58pub struct AutomaticStatisticsStatus {
59    pub running: bool,
60    pub completed: u64,
61    pub last_error: Option<String>,
62}
63
64#[derive(Default)]
65pub(crate) struct AutomaticStatistics {
66    clients: AtomicUsize,
67    control: Mutex<Option<StatisticsWorker>>,
68    status: Mutex<AutomaticStatisticsStatus>,
69}
70
71struct StatisticsWorker {
72    sender: mpsc::SyncSender<()>,
73    cancellation: uqa_core::CancellationToken,
74    thread: std::thread::JoinHandle<()>,
75}
76
77pub(crate) fn now_ms() -> u64 {
78    SystemTime::now()
79        .duration_since(UNIX_EPOCH)
80        .map_or(0, |duration| {
81            u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
82        })
83}
84
85impl Engine {
86    pub fn automatic_statistics_status(&self) -> AutomaticStatisticsStatus {
87        self.statistics.automatic_statistics.status.lock().clone()
88    }
89
90    pub fn automatic_diskann_maintenance_status(&self) -> crate::DiskANNMaintenanceStatus {
91        self.statistics.diskann.lock().clone()
92    }
93
94    /// Database-shared, process-local rebuild thresholds for future maintenance admissions.
95    pub fn diskann_rebuild_policy(&self) -> crate::DiskANNRebuildPolicy {
96        *self.statistics.diskann_policy.lock()
97    }
98
99    /// Revisit pending changes under this policy at the next maintenance poll. Current builds retain their admitted policy and resource limits; this setting is not persisted.
100    pub fn set_diskann_rebuild_policy(&self, policy: crate::DiskANNRebuildPolicy) {
101        *self.statistics.diskann_policy.lock() = policy;
102        self.start_automatic_statistics();
103    }
104
105    pub(crate) fn record_statistics_change(
106        &self,
107        name: &str,
108        table: &TableState,
109        count: u64,
110    ) -> StorageBackendResult<()> {
111        if self.storage.catalog.is_none()
112            || table.persistence == uqa_sql::ast::RelationPersistence::Temporary
113        {
114            let previous_rows = table
115                .column_stats
116                .read()
117                .values()
118                .next()
119                .map(|stats| stats.row_count);
120            return table.statistics_maintenance.lock().record_changes(
121                table.object_id(),
122                count,
123                previous_rows,
124                now_ms(),
125            );
126        }
127        let mut stack = self.session.transactions.lock();
128        if let Some(frame) = stack.last_mut() {
129            let change = frame.statistics_changes.entry(name.to_owned()).or_default();
130            if change.object_id != table.object_id() {
131                *change = StatisticsChange {
132                    object_id: table.object_id(),
133                    count: 0,
134                    reset: false,
135                    through: None,
136                };
137            }
138            change.count = change.count.saturating_add(count);
139            return Ok(());
140        }
141        drop(stack);
142        let changes = BTreeMap::from([(
143            name.to_owned(),
144            StatisticsChange {
145                object_id: table.object_id(),
146                count,
147                reset: false,
148                through: None,
149            },
150        )]);
151        let settlement = self.persist_statistics_changes(&changes)?;
152        self.settle_statistics_changes(settlement);
153        self.start_automatic_statistics();
154        Ok(())
155    }
156
157    /// Write the maintenance records the changes of a committing transaction call for. A session keeps the changes that decide nothing ([`MaintenanceState::defers`]) to itself and records them with a later commit, where an analysis learns of every committed write from the table's data generation instead of from this record.
158    pub(crate) fn persist_statistics_changes(
159        &self,
160        changes: &StatisticsChanges,
161    ) -> StorageBackendResult<StatisticsSettlement> {
162        let mut settlement = StatisticsSettlement::default();
163        let Some(catalog) = self.storage.catalog.as_deref() else {
164            return Ok(settlement);
165        };
166        // A session keeps changes only where an analysis can do without the record: cache generations tell it of every committed write to what it sampled, and numbered commits tell the session which of its changes the analysis saw.
167        let keeps = self.epochs.storage_cache_revisions.lock().is_some()
168            && self
169                .storage
170                .backend
171                .as_ref()
172                .is_some_and(|backend| backend.transaction_model().is_versioned());
173        let kept = self.session.kept_statistics.lock().clone();
174        let now = now_ms();
175        let tables = self.storage.tables.read();
176        for (name, change) in changes {
177            if change.count == 0 {
178                continue;
179            }
180            // DROP/recreate and rename cannot attach the old relation's
181            // pending maintenance to a different durable object.
182            let Some(table) = tables.iter().find_map(|(relation, table)| {
183                (relation.qualified_name() == *name && table.object_id() == change.object_id)
184                    .then_some(table)
185            }) else {
186                continue;
187            };
188            let mut state = MaintenanceState::load_for(catalog, name, change.object_id)?;
189            // What the session kept counts unless an analysis has sampled the commits that made it.
190            let earlier = kept
191                .get(name)
192                .filter(|earlier| {
193                    earlier.object_id == change.object_id && !state.covers(earlier.through)
194                })
195                .map_or(0, |earlier| earlier.count);
196            let count = earlier.saturating_add(change.count);
197            if keeps && state.defers(count, now) {
198                settlement.kept.insert(
199                    name.clone(),
200                    StatisticsChange {
201                        object_id: change.object_id,
202                        count,
203                        reset: false,
204                        through: None,
205                    },
206                );
207                continue;
208            }
209            state.record_changes(
210                change.object_id,
211                count,
212                table
213                    .column_stats
214                    .read()
215                    .values()
216                    .next()
217                    .map(|stats| stats.row_count),
218                now,
219            )?;
220            state.save(catalog, name)?;
221            settlement.recorded.push(name.clone());
222        }
223        Ok(settlement)
224    }
225
226    /// Take over what a successful commit did with its statistics changes. The caller has left the transaction that committed them.
227    pub(crate) fn settle_statistics_changes(&self, settlement: StatisticsSettlement) {
228        // Read after the commit, so that it lies at or after every commit of the kept changes. A session that is still inside a transaction would read that transaction's earlier view.
229        let through = (!settlement.kept.is_empty())
230            .then(|| {
231                let backend = self.storage.backend.as_ref()?;
232                if backend.in_transaction() {
233                    return None;
234                }
235                backend.change_version().ok()?
236            })
237            .flatten();
238        let mut kept = self.session.kept_statistics.lock();
239        for name in settlement.recorded {
240            kept.remove(&name);
241        }
242        kept.extend(settlement.kept.into_iter().map(|(name, mut change)| {
243            change.through = through;
244            (name, change)
245        }));
246    }
247
248    /// The commit sequence an analysis that samples now reads at, for the record it writes. `None` where commits are not numbered, and in a transaction that reads its data at an earlier fixed snapshot than its storage view.
249    pub(crate) fn statistics_sample_sequence(&self) -> Option<u64> {
250        let backend = self.storage.backend.as_ref()?;
251        if !backend.transaction_model().is_versioned()
252            || self.current_transaction_uses_fixed_snapshot()
253        {
254            return None;
255        }
256        backend.change_version().ok().flatten()
257    }
258
259    pub(crate) fn merge_statistics_changes(
260        into: &mut StatisticsChanges,
261        changes: StatisticsChanges,
262    ) {
263        for (name, change) in changes {
264            let previous = into.entry(name).or_default();
265            if previous.object_id == change.object_id && !change.reset {
266                previous.count = previous.count.saturating_add(change.count);
267            } else {
268                *previous = change;
269            }
270        }
271    }
272
273    pub(crate) fn clear_pending_statistics_changes(&self, name: &str, object_id: [u8; 16]) {
274        if let Some(frame) = self.session.transactions.lock().last_mut() {
275            // A child analysis covers its parent's pending changes only if the child commits. Preserve the parent entry across a child rollback.
276            frame.statistics_changes.insert(
277                name.to_owned(),
278                StatisticsChange {
279                    object_id,
280                    count: 0,
281                    reset: true,
282                    through: None,
283                },
284            );
285        }
286    }
287
288    pub(crate) fn start_automatic_statistics(&self) {
289        let Some(provider) = self.storage.provider.as_ref() else {
290            return;
291        };
292        if self.session.statistics_worker.load(Ordering::Acquire) {
293            return;
294        }
295        let automatic = &self.statistics.automatic_statistics;
296        if !self.session.statistics_client.swap(true, Ordering::AcqRel) {
297            automatic.clients.fetch_add(1, Ordering::AcqRel);
298        }
299        let mut control = automatic.control.lock();
300        if let Some(worker) = control.as_ref() {
301            // Session admission only retains the worker. Sending a liveness
302            // probe wakes its wait without advancing the maintenance deadline.
303            if !worker.thread.is_finished() {
304                return;
305            }
306            *control = None;
307        }
308        let (requests, receiver) = mpsc::sync_channel(1);
309        let provider = Arc::downgrade(provider);
310        let manager = Arc::downgrade(&self.row_locks);
311        let statistics = Arc::downgrade(&self.statistics);
312        let cancellation = uqa_core::CancellationToken::new();
313        let worker_cancellation = cancellation.clone();
314        match std::thread::Builder::new()
315            .name("uqa-maintenance".into())
316            .spawn(move || {
317                worker::run(
318                    &provider,
319                    &manager,
320                    &statistics,
321                    &receiver,
322                    &worker_cancellation,
323                );
324            }) {
325            Ok(thread) => {
326                *control = Some(StatisticsWorker {
327                    sender: requests,
328                    cancellation,
329                    thread,
330                });
331            }
332            Err(error) => {
333                automatic.status.lock().last_error = Some(error.to_string());
334            }
335        }
336    }
337
338    pub(crate) fn release_automatic_statistics_client(&self) {
339        if self.session.statistics_client.swap(false, Ordering::AcqRel) {
340            let automatic = &self.statistics.automatic_statistics;
341            if automatic.clients.fetch_sub(1, Ordering::AcqRel) != 1 {
342                return;
343            }
344            let worker = {
345                let mut control = automatic.control.lock();
346                if automatic.clients.load(Ordering::Acquire) != 0 {
347                    return;
348                }
349                control.take()
350            };
351            if let Some(worker) = worker {
352                worker.cancellation.cancel();
353                let _ = worker.sender.try_send(());
354                // Release every maintenance connection before the final
355                // client's drop returns. redb requires this for immediate
356                // reopen; SQLite requires it for final WAL checkpointing.
357                if worker.thread.join().is_err() {
358                    automatic.status.lock().last_error =
359                        Some("automatic statistics worker panicked".into());
360                }
361            }
362        }
363    }
364}