uqa-engine 0.6.0

Engine: schema-aware table store, catalog restore, transactions
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Refresh only the dependencies changed in a pinned durable snapshot.

use std::collections::{BTreeMap, BTreeSet};
use std::sync::atomic::Ordering;

use uqa_storage::{CatalogCacheRevisions, StorageBackendError, StorageBackendResult};

use super::Engine;

fn changed_names(
    previous: &BTreeMap<String, u64>,
    current: &BTreeMap<String, u64>,
) -> BTreeSet<String> {
    previous
        .keys()
        .chain(current.keys())
        .filter(|name| previous.get(*name) != current.get(*name))
        .cloned()
        .collect()
}

fn observed_private_data(
    name: &str,
    current: &CatalogCacheRevisions,
    previous_view: Option<&uqa_storage::key_value::KeyValueReadRevision>,
) -> bool {
    current
        .table_data
        .get(name)
        .is_some_and(|value| CatalogCacheRevisions::is_private_generation(*value))
        && current
            .table_data_commits
            .get(name)
            .is_some_and(|(database, sequence)| {
                previous_view.is_some_and(|view| view.includes_commit(*database, *sequence))
            })
}

impl Engine {
    /// The successful statement boundary certifies that Engine's adapters finished applying their private definitions. Record only the exact current view and only while its committed base is already observed; the next command still detects external commits and undo.
    pub(crate) fn retain_completed_statement_catalog(&self) {
        if !self.epochs.table_catalog.dirty.load(Ordering::Acquire)
            && !self.epochs.catalog_registry.dirty.load(Ordering::Acquire)
        {
            return;
        }
        let Some(backend) = self.storage.backend.as_ref().filter(|backend| {
            backend.transaction_model().is_versioned() && backend.in_transaction()
        }) else {
            return;
        };
        let Ok(Some(current)) = backend.read_view_revision() else {
            return;
        };
        let mut seen = self.epochs.seen_storage_read_view.lock();
        if seen
            .as_ref()
            .is_some_and(|previous| current.same_committed_state(previous))
        {
            *seen = Some(current);
        }
    }

    /// After this session's own data commit, keep its table caches when that commit is the only one since the view they reflect. Its writes already maintained them, so the next statement need not rebuild them as it must after another writer. Any other commit, or a catalog, registry, graph or schema revision change, leaves the observed state behind for the ordinary refresh. A catalog that reports no cache revisions, before as after, relies on the caller, which adopts only a commit whose dirty state shows data changes alone. A failure to observe the committed view has the same effect as another commit: the next statement's refresh reads it again and reports any persistent error.
    pub(crate) fn adopt_own_commit_revisions(&self) {
        self.adopt_own_transaction_revisions(false);
    }

    /// Retain definitions maintained by a completed Engine DDL transaction only when its commit alone follows the cached committed base. The ordinary refresh still handles an intervening peer commit, unknown history or failed observation.
    pub(crate) fn adopt_own_catalog_commit_revisions(&self) {
        self.adopt_own_transaction_revisions(true);
    }

    fn adopt_own_transaction_revisions(&self, retain_definitions: bool) {
        let Some(backend) = self.storage.backend.as_ref() else {
            return;
        };
        let Some(catalog) = self.storage.catalog.as_ref() else {
            return;
        };
        let previous_view = self.epochs.seen_storage_read_view.lock().clone();
        let Some(previous_view) = previous_view else {
            return;
        };
        if backend.in_transaction() {
            return;
        }
        // Observed before the view: a sibling that publishes later remains unobserved.
        let data_epoch = self.epochs.table_data.published.load(Ordering::Acquire);
        let catalog_epoch = self.epochs.table_catalog.published.load(Ordering::Acquire);
        let registry_epoch = self
            .epochs
            .catalog_registry
            .published
            .load(Ordering::Acquire);
        if !retain_definitions {
            if let Ok(Some(committed)) = backend.committed_data_revision() {
                if committed.revision.follows_by_one_commit(&previous_view) {
                    // The provider certified that this exact commit preserved definitions. Retain the older data-generation baseline: a later foreign commit can then cause an extra refresh, but cannot hide a changed table. The receipt is this session's view, never a claim to have observed a later writer.
                    self.adopt_committed_view(
                        committed.revision,
                        Some(committed.change_version),
                        data_epoch,
                    );
                    return;
                }
            }
        }
        let previous = self.epochs.storage_cache_revisions.lock().clone();
        // A versioned session's pinned view reports its own change version, which costs no further read of the latest commit. Any other session asks the monitor first: inside its read transaction the monitor could wait behind a writer that waits for this reader.
        let pinned_version = backend.transaction_model().is_versioned();
        let before = if pinned_version {
            None
        } else {
            match backend.change_version() {
                Ok(version) => Some(version),
                Err(_) => return,
            }
        };
        if backend.begin_read_transaction().is_err() {
            return;
        }
        let observed = before
            .map_or_else(|| backend.change_version(), Ok)
            .and_then(|version| {
                Ok((
                    version,
                    backend.read_view_revision()?,
                    catalog.cache_revisions()?,
                ))
            });
        if backend.rollback_transaction().is_err() {
            return;
        }
        let Ok((version, Some(view), current)) = observed else {
            return;
        };
        if !view.follows_by_one_commit(&previous_view) {
            return;
        }
        match (&previous, &current) {
            (Some(previous), Some(current)) => {
                if !retain_definitions
                    && (current.table_catalog != previous.table_catalog
                        || current.registries != previous.registries
                        || current.graphs != previous.graphs
                        || current.storage_schema != previous.storage_schema)
                {
                    return;
                }
            }
            (None, None) => {}
            _ => return,
        }
        *self.epochs.storage_cache_revisions.lock() = current;
        self.adopt_committed_view(view, version, data_epoch);
        if retain_definitions {
            self.epochs
                .table_catalog
                .seen
                .store(catalog_epoch, Ordering::Release);
            self.epochs
                .catalog_registry
                .seen
                .store(registry_epoch, Ordering::Release);
        }
    }

    fn adopt_committed_view(
        &self,
        view: uqa_storage::key_value::KeyValueReadRevision,
        version: Option<u64>,
        data_epoch: u64,
    ) {
        *self.epochs.seen_storage_read_view.lock() = Some(view);
        if let Some(version) = version {
            self.epochs
                .seen_storage_change_version
                .store(version, Ordering::Release);
        }
        self.epochs
            .table_data
            .seen
            .store(data_epoch, Ordering::Release);
        self.clear_bayesian_params_cache();
        self.invalidate_prepared_plans();
    }

    /// An epoch may be published after the physical commit was already seen.
    /// Reconcile against the durable generations rather than decoding again.
    pub(super) fn refresh_tracked_storage_snapshot(&self) -> StorageBackendResult<bool> {
        if self.epochs.storage_cache_revisions.lock().is_none() {
            return Ok(false);
        }
        let Some(backend) = self.storage.backend.as_ref() else {
            return Ok(false);
        };
        let _statement = self.lock_statement_gate();
        let _refresh = self.epochs.external_commit_refresh.lock();
        if backend.in_transaction() {
            return Ok(false);
        }
        backend.begin_read_transaction()?;
        let result = self.refresh_pinned_transaction_snapshot();
        let cleanup = backend.rollback_transaction();
        match (result, cleanup) {
            (Ok(()), Ok(())) => Ok(true),
            (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
            (Err(error), Err(cleanup)) => Err(StorageBackendError::Other(format!(
                "cache refresh failed: {error}; snapshot cleanup failed: {cleanup}"
            ))),
        }
    }

    pub(super) fn refresh_tracked_pinned_snapshot(
        &self,
        catalog_epoch: u64,
        data_epoch: u64,
        registry_epoch: u64,
        committed_unchanged: bool,
        previous_view: Option<&uqa_storage::key_value::KeyValueReadRevision>,
    ) -> StorageBackendResult<bool> {
        let Some(catalog) = self.storage.catalog.as_ref() else {
            return Ok(false);
        };
        let Some(current) = catalog.cache_revisions()? else {
            return Ok(false);
        };
        let previous = self.epochs.storage_cache_revisions.lock().clone();
        let catalog_changed = previous.as_ref().is_none_or(|previous| {
            previous.table_catalog != current.table_catalog
                || previous.storage_schema != current.storage_schema
        });
        let registries_changed = catalog_changed
            || previous
                .as_ref()
                .is_none_or(|previous| previous.registries != current.registries);
        if catalog_changed {
            self.clear_persistent_table_bindings_for_catalog_reload();
            self.reload_table_catalog(catalog_epoch)?;
            self.synchronize_partition_identity_watermarks()?;
        }
        if registries_changed {
            self.reload_catalog_registries(registry_epoch)?;
        } else if let Some(graphs) = current.graphs.as_ref() {
            let changed = self.refresh_graph_handles(
                catalog.as_ref(),
                previous
                    .as_ref()
                    .and_then(|previous| previous.graphs.as_ref()),
                graphs,
            )?;
            if !changed.is_empty() {
                // Graph namespaces and AGE label relations also appear in
                // regnamespace/regclass output, independently of SQL DDL.
                self.clear_regtype_output_cache();
            }
            self.refresh_changed_graph_path_indexes(catalog.as_ref(), &changed)?;
        }
        // Physical indexes and analyzer bindings belong to the same committed revision. Restore changed registries before reopening data stores, so an old cached binding never meets newly committed occurrence metadata.
        if !catalog_changed {
            if let Some(previous) = previous.as_ref() {
                self.refresh_changed_table_caches(
                    previous,
                    &current,
                    committed_unchanged,
                    previous_view,
                )?;
            }
        }
        // Generations become observed only after every dependent cache was
        // restored successfully. An error leaves the snapshot eligible to retry.
        self.epochs
            .table_catalog
            .seen
            .store(catalog_epoch, Ordering::Release);
        self.epochs
            .table_data
            .seen
            .store(data_epoch, Ordering::Release);
        self.epochs
            .catalog_registry
            .seen
            .store(registry_epoch, Ordering::Release);
        if previous.as_ref() != Some(&current) {
            if catalog_changed || registries_changed {
                self.clear_sql_statement_cache();
            } else {
                self.invalidate_optimized_sql_plans();
            }
            self.clear_bayesian_params_cache();
            self.invalidate_prepared_plans();
        }
        *self.epochs.storage_cache_revisions.lock() = Some(current);
        Ok(true)
    }

    /// `committed_unchanged` reports that the committed state is the one the caches were last refreshed from, so a private generation is this transaction's own change, which its writes already applied to the caches. A rollback restores the committed generation, which differs from the private one and refreshes the table.
    fn refresh_changed_table_caches(
        &self,
        previous: &CatalogCacheRevisions,
        current: &CatalogCacheRevisions,
        committed_unchanged: bool,
        previous_view: Option<&uqa_storage::key_value::KeyValueReadRevision>,
    ) -> StorageBackendResult<()> {
        let foreign = |previous: &BTreeMap<String, u64>, current: &BTreeMap<String, u64>| {
            let mut names = changed_names(previous, current);
            if committed_unchanged {
                names.retain(|name| {
                    !current.get(name).is_some_and(|generation| {
                        CatalogCacheRevisions::is_private_generation(*generation)
                    })
                });
            }
            names
        };
        let mut data = foreign(&previous.table_data, &current.table_data);
        // Own commit adoption can leave the generation baseline behind. Compare
        // the table's actual last committed change with the view the caches
        // reflect, rather than mistaking our own older commit for a peer write.
        data.retain(|name| !observed_private_data(name, current, previous_view));
        let statistics = foreign(&previous.column_statistics, &current.column_statistics);
        let maintenance = changed_names(
            &previous.statistics_maintenance,
            &current.statistics_maintenance,
        );
        let names = data
            .iter()
            .chain(&statistics)
            .chain(&maintenance)
            .cloned()
            .collect::<BTreeSet<_>>();
        let catalog = self
            .storage
            .catalog
            .as_ref()
            .expect("tracked persistent catalog");
        let private_snapshot = self
            .storage
            .backend
            .as_ref()
            .map(|backend| backend.transaction_has_written())
            .transpose()?
            .unwrap_or(false);
        let mut partition_watermarks_changed = false;
        for name in names {
            let relation = uqa_storage::RelationIdentity::from_legacy_name(&name)
                .map_err(StorageBackendError::Other)?;
            let table = self.storage.tables.read().get(&relation).cloned();
            let Some(table) = table else {
                continue;
            };
            if table.persistence == uqa_sql::ast::RelationPersistence::Temporary {
                continue;
            }
            if data.contains(&name) {
                self.rebind_persistent_table_stores(&name, &table)?;
                self.refresh_table_next_id(&name, &table)?;
                table.doc_count_dirty.store(true, Ordering::Release);
                let hierarchy = table.hierarchy.read();
                partition_watermarks_changed |= (hierarchy.partition_spec.is_some()
                    || hierarchy.is_partition())
                    && table.columns.read().iter().any(|column| {
                        column
                            .auto_increment
                            .as_ref()
                            .is_some_and(uqa_sql::ast::AutoIncrement::is_legacy)
                    });
            }
            if statistics.contains(&name) {
                let load = || Self::load_column_stats_from_catalog(catalog.as_ref(), &name);
                // A private transaction may reuse a durable counter value
                // after rollback. Never publish its payload to sibling sessions.
                let stats = if private_snapshot {
                    std::sync::Arc::new(load()?)
                } else {
                    self.statistics.statistics_snapshots.load(
                        &name,
                        table.object_id(),
                        current
                            .column_statistics
                            .get(&name)
                            .copied()
                            .unwrap_or_default(),
                        load,
                    )?
                };
                table.column_stats.restore(&stats);
                table.column_stats_loaded.store(true, Ordering::Release);
            }
            let dirty = (table.column_stats.read().is_empty() && !table.columns.read().is_empty())
                || crate::statistics::MaintenanceState::load_for(
                    catalog.as_ref(),
                    &name,
                    table.object_id(),
                )?
                .invalidates_existing_statistics();
            table.column_stats_dirty.store(dirty, Ordering::Release);
        }
        if partition_watermarks_changed {
            self.synchronize_partition_identity_watermarks()?;
        }
        Ok(())
    }
}