1mod admission;
10mod codec;
11mod commit_cache;
12mod connection_functions;
13mod identifiers;
14mod key_value;
15#[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
16pub(crate) mod leases;
17pub mod native;
18mod read;
19pub(crate) mod receipts;
20mod reclamation;
21mod resources;
22pub(crate) mod restore;
23mod retention;
24mod runs;
25mod schema;
26mod serializable;
27mod synchronization;
28#[cfg(test)]
29mod tests;
30mod tombstones;
31mod version_metadata;
32mod write;
33
34pub(crate) use schema::WritePermit;
35pub use serializable::SQLiteSerializableAdmission;
36
37use std::sync::Arc;
38
39use rusqlite::Connection;
40use uqa_storage::mvcc::{
41 CommitResult, CommitStatus, CommittedRecordSnapshot, DatabaseId, PreparedRecordCommit,
42 StorageTransactionId, VersionError, VersionResult, VersionedPersistence,
43};
44use uqa_storage::read_control::StorageReadControl;
45
46use crate::{ManagedConnection, SQLiteError};
47
48#[derive(Debug, thiserror::Error)]
49enum Error {
50 #[error(transparent)]
51 SQLite(#[from] rusqlite::Error),
52 #[error(transparent)]
53 Version(#[from] VersionError),
54}
55
56impl Error {
57 fn into_version(self) -> VersionError {
58 match self {
59 Self::SQLite(error) => sqlite_error(error),
60 Self::Version(error) => error,
61 }
62 }
63}
64
65impl From<SQLiteError> for Error {
66 fn from(error: SQLiteError) -> Self {
67 match error {
68 SQLiteError::Memory(error) => Self::Version(VersionError::Memory(error)),
69 SQLiteError::Cancelled(error) => Self::Version(VersionError::Cancelled(error)),
70 SQLiteError::SQLite(error) => Self::SQLite(error),
71 error => Self::Version(VersionError::Storage(error.into())),
72 }
73 }
74}
75
76type PhysicalResult<T> = std::result::Result<T, Error>;
77
78fn sqlite_error(error: rusqlite::Error) -> VersionError {
79 VersionError::Storage(SQLiteError::SQLite(error).into())
80}
81
82#[derive(Clone)]
86pub struct SQLiteRecordStore {
87 connection: ManagedConnection,
88 identity: DatabaseId,
89 native: Option<native::NativeRecordNamespace>,
90 snapshots: Arc<uqa_storage::mvcc::SnapshotRegistry>,
91 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
92 receipt_leases: receipts::ReceiptLeaseFile,
93}
94
95#[cfg(test)]
96thread_local! {
97 static READ_VALIDATIONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
99 static RECORD_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
101}
102
103impl SQLiteRecordStore {
104 pub(crate) fn has_native_mapping(connection: &Connection) -> VersionResult<bool> {
105 native::present(connection).map_err(Error::into_version)
106 }
107
108 pub(crate) fn initialize_native_in(
110 connection: &ManagedConnection,
111 transaction: &Connection,
112 control: &StorageReadControl,
113 ) -> VersionResult<Self> {
114 let native::NativeMapping {
115 identity,
116 namespace,
117 } = native::initialize_in(transaction, control).map_err(Error::into_version)?;
118 Ok(Self {
119 snapshots: retention::registry(connection, identity)?,
120 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
121 receipt_leases: receipts::ReceiptLeaseFile::default(),
122 connection: connection.record_connection(),
123 identity,
124 native: Some(namespace),
125 })
126 }
127
128 pub fn new(connection: &ManagedConnection) -> VersionResult<Self> {
129 if connection.in_transaction() {
130 return Err(VersionError::Storage(
131 SQLiteError::TransactionAlreadyActive.into(),
132 ));
133 }
134 let connection = connection.record_connection();
135 let identity = connection
136 .with(|connection| Ok(schema::initialize(connection)))
137 .map_err(|error| VersionError::Storage(error.into()))?
138 .map_err(Error::into_version)?;
139 Ok(Self {
140 snapshots: retention::registry(&connection, identity)?,
141 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
142 receipt_leases: receipts::ReceiptLeaseFile::default(),
143 connection,
144 identity,
145 native: None,
146 })
147 }
148
149 pub fn for_native(
151 connection: &ManagedConnection,
152 control: &StorageReadControl,
153 ) -> VersionResult<Self> {
154 if connection.in_transaction() {
155 return Err(VersionError::Storage(
156 SQLiteError::TransactionAlreadyActive.into(),
157 ));
158 }
159 let connection = connection.record_connection();
160 let native::NativeMapping {
161 identity,
162 namespace,
163 } = connection
164 .with(|connection| Ok(native::initialize(connection, control)))
165 .map_err(|error| VersionError::Storage(error.into()))?
166 .map_err(Error::into_version)?;
167 Ok(Self {
168 snapshots: retention::registry(&connection, identity)?,
169 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
170 receipt_leases: receipts::ReceiptLeaseFile::default(),
171 connection,
172 identity,
173 native: Some(namespace),
174 })
175 }
176
177 pub(crate) fn for_key_value(
178 connection: &ManagedConnection,
179 control: &StorageReadControl,
180 ) -> VersionResult<Self> {
181 let connection = connection.record_connection();
182 let identity = connection
183 .with(|connection| Ok(key_value::initialize(connection, control)))
184 .map_err(|error| VersionError::Storage(error.into()))?
185 .map_err(Error::into_version)?;
186 Ok(Self {
187 snapshots: retention::registry(&connection, identity)?,
188 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
189 receipt_leases: receipts::ReceiptLeaseFile::default(),
190 connection,
191 identity,
192 native: None,
193 })
194 }
195
196 #[must_use]
198 pub fn native_namespace(&self) -> Option<DatabaseId> {
199 self.native.map(|namespace| namespace.0)
200 }
201
202 fn with<T>(
203 &self,
204 operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
205 ) -> VersionResult<T> {
206 self.connection
207 .with(|connection| Ok(operation(connection)))
208 .map_err(|error| VersionError::Storage(error.into()))?
209 .map_err(Error::into_version)
210 }
211
212 fn read<T>(
214 &self,
215 operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
216 ) -> VersionResult<T> {
217 #[cfg(test)]
218 RECORD_READS.with(|count| count.set(count.get() + 1));
219 self.connection
220 .with_record_read(|connection, validated| {
221 Ok((|| {
222 let transaction = connection.unchecked_transaction()?;
223 let state = crate::connection::ValidatedRead {
224 store: (
225 self.identity.as_bytes(),
226 self.native.map(|namespace| namespace.0.as_bytes()),
227 ),
228 data_version: transaction
229 .prepare_cached("PRAGMA data_version")?
230 .query_row([], |row| row.get(0))?,
231 changes: connection.total_changes(),
232 };
233 if validated.get() != Some(state) {
234 native::check_mapping(&transaction, self.native)?;
235 codec::header(&transaction, self.identity)?;
236 validated.set(Some(state));
237 #[cfg(test)]
238 READ_VALIDATIONS.with(|count| count.set(count.get() + 1));
239 }
240 let result = operation(&transaction)?;
241 transaction.commit()?;
242 Ok(result)
243 })())
244 })
245 .map_err(|error| VersionError::Storage(error.into()))?
246 .map_err(Error::into_version)
247 }
248
249 fn with_write<T>(
250 &self,
251 control: &StorageReadControl,
252 operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
253 ) -> VersionResult<T> {
254 let mut operation = Some(operation);
255 loop {
256 control.cancellation().check()?;
257 let result = self
258 .connection
259 .with_record_connection(control, |connection| {
260 let _timeout = admission::BusyTimeout::new(connection)?;
261 control.cancellation().check()?;
262 Ok(operation.take().expect("record write runs once")(
263 connection,
264 ))
265 });
266 match result {
267 Err(SQLiteError::SQLite(error))
268 if operation.is_some() && admission::is_busy(&error) =>
269 {
270 admission::wait(control).map_err(Error::into_version)?;
272 }
273 result => {
274 return result
275 .map_err(|error| Error::from(error).into_version())?
276 .map_err(Error::into_version);
277 }
278 }
279 }
280 }
281
282 fn check_transaction(&self, transaction: StorageTransactionId) -> VersionResult<()> {
283 if transaction.database() != self.identity {
284 return Err(VersionError::WrongDatabase);
285 }
286 Ok(())
287 }
288}
289
290impl VersionedPersistence for SQLiteRecordStore {
291 fn diskann_population_record_layout(
292 &self,
293 ) -> Option<&dyn uqa_storage::mvcc::DiskANNPopulationRecordLayout> {
294 Some(match &self.native {
295 Some(namespace) => namespace,
296 None => &uqa_storage::key_value::KeyValueDiskANNPopulationRecords,
297 })
298 }
299
300 fn resource_leases(&self) -> Option<&dyn uqa_storage::mvcc::ResourceLeaseProvider> {
301 Some(self)
302 }
303
304 fn serializable_coordinator(&self) -> Option<&dyn uqa_storage::mvcc::SerializableCoordinator> {
305 Some(self)
306 }
307
308 fn identifier_watermark(
309 &self,
310 namespace: &[u8],
311 control: &StorageReadControl,
312 ) -> VersionResult<Option<u64>> {
313 control.cancellation().check()?;
314 self.with(|connection| {
315 identifiers::read(connection, self.identity, self.native, namespace, control)
316 })
317 }
318
319 fn allocate_identifiers(
320 &self,
321 namespace: &[u8],
322 request: uqa_storage::mvcc::IdentifierRequest,
323 control: &StorageReadControl,
324 ) -> VersionResult<uqa_storage::mvcc::IdentifierAllocation> {
325 control.cancellation().check()?;
326 self.with_write(control, |connection| {
327 identifiers::allocate(
328 connection,
329 self.identity,
330 self.native,
331 namespace,
332 request,
333 control,
334 )
335 })
336 }
337
338 fn database_id(&self) -> DatabaseId {
339 self.identity
340 }
341 fn graph_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::GraphRecordLayout> {
342 match self.native.as_ref() {
343 Some(namespace) => Some(namespace),
344 None => Some(&uqa_storage::key_value::KeyValueGraphRecords),
345 }
346 }
347 fn occurrence_record_layout(&self) -> &dyn uqa_storage::mvcc::OccurrenceRecordLayout {
348 if self.native.is_some() {
349 &crate::inverted_index::NativeOccurrenceRecords
350 } else {
351 &uqa_storage::key_value::KeyValueOccurrenceRecords
352 }
353 }
354 fn notification_record_layout(&self) -> &dyn uqa_storage::mvcc::NotificationRecordLayout {
355 match self.native.as_ref() {
356 Some(namespace) => namespace,
357 None => &uqa_storage::key_value::KeyValueNotificationRecords,
358 }
359 }
360
361 fn maintenance_record_layout(&self) -> &dyn uqa_storage::mvcc::MaintenanceRecordLayout {
362 if self.native.is_some() {
363 &native::NativeMaintenanceRecords
364 } else {
365 &uqa_storage::key_value::KeyValueMaintenanceRecords
366 }
367 }
368 fn ivf_record_layout(&self) -> &dyn uqa_storage::mvcc::IVFRecordLayout {
369 if self.native.is_some() {
370 &crate::vector_index::NativeIVFRecords
371 } else {
372 &uqa_storage::key_value::KeyValueIVFRecords
373 }
374 }
375 fn hnsw_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::HNSWRecordLayout> {
376 if self.native.is_some() {
377 Some(&crate::vector_index::NativeHNSWRecords)
378 } else {
379 Some(&uqa_storage::key_value::KeyValueHNSWRecords)
380 }
381 }
382 fn allocate_transaction(
383 &self,
384 control: &StorageReadControl,
385 ) -> VersionResult<StorageTransactionId> {
386 control.cancellation().check()?;
387 self.with_write(control, |connection| {
388 write::allocate(connection, self.identity, self.native, control)
389 })
390 }
391
392 fn allocate_managed_transaction(
393 &self,
394 control: &StorageReadControl,
395 ) -> VersionResult<uqa_storage::mvcc::RetainedTransactionAllocation> {
396 self.allocate_receipt_owner(control)
397 }
398 fn snapshot(
399 &self,
400 control: &StorageReadControl,
401 ) -> VersionResult<Arc<dyn CommittedRecordSnapshot>> {
402 control.cancellation().check()?;
403 let monitor = self.commit_monitor_version()?;
405 let mut reclamation_epoch = 0;
406 let lease = self.snapshots.capture(control, || {
407 self.with(|connection| {
408 let read = connection.unchecked_transaction()?;
409 native::check_mapping(&read, self.native)?;
410 let sequence = codec::header(&read, self.identity)?.sequence;
411 reclamation_epoch = tombstones::epoch(&read)?;
412 read.commit()?;
413 Ok(sequence)
414 })
415 })?;
416 uqa_storage::mvcc::retain_record_snapshot(
417 read::Snapshot {
418 store: self.clone(),
419 sequence: lease.sequence(),
420 reclamation_epoch,
421 monitor: monitor.map(std::sync::atomic::AtomicU64::new),
422 table_owners: read::table_owners::TableOwners::default(),
423 row_presence: read::row_presence::RowPresence::default(),
424 _lease: lease,
425 },
426 control,
427 )
428 }
429 fn commit_monitor_version(&self) -> VersionResult<Option<u64>> {
430 self.connection
431 .commit_monitor_version()
432 .map_err(|error| VersionError::Storage(error.into()))
433 }
434 fn reclaim_versions(&self, control: &StorageReadControl) -> VersionResult<u64> {
435 self.snapshots.reclaim(control, |oldest| {
436 self.with_write(control, |connection| {
437 reclamation::reclaim(connection, self.identity, self.native, oldest, control)
438 })
439 })
440 }
441
442 fn reclaim_tombstones(
443 &self,
444 request: &uqa_storage::mvcc::TombstoneReclamationRequest<'_>,
445 control: &StorageReadControl,
446 ) -> VersionResult<uqa_storage::mvcc::TombstoneReclamationStep> {
447 request.validate(control)?;
448 self.snapshots.reclaim(control, |oldest| {
449 if oldest.is_some() {
450 return Ok(uqa_storage::mvcc::TombstoneReclamationStep::Retained);
451 }
452 self.with_write(control, |connection| {
453 tombstones::reclaim(connection, self.identity, self.native, request, control)
454 })
455 })
456 }
457
458 fn reclaim_diskann_tombstones(&self, control: &StorageReadControl) -> VersionResult<()> {
459 if self.native.is_none() {
460 return uqa_storage::mvcc::reclaim_key_value_diskann_tombstones(self, control);
461 }
462 for family in [
463 native::NativeRecordFamily::DiskANNRecords,
464 native::NativeRecordFamily::VectorOrigins,
465 native::NativeRecordFamily::VectorChanges,
466 native::NativeRecordFamily::VectorPopulations,
467 native::NativeRecordFamily::VectorPopulationWitnesses,
468 ] {
469 let prefix = native::NativeRecordIdentity::family_prefix(family, control)?;
470 uqa_storage::mvcc::reclaim_tombstone_prefix(self, &prefix, control)?;
471 }
472 let prefix = self.vector_field_guard_layout().prefix(control)?;
473 uqa_storage::mvcc::reclaim_tombstone_prefix(self, &prefix, control)?;
474 Ok(())
475 }
476
477 fn vector_field_guard_layout(&self) -> &dyn uqa_storage::mvcc::VectorFieldGuardLayout {
478 self
479 }
480
481 fn commit(
482 &self,
483 transaction: StorageTransactionId,
484 prepared: &PreparedRecordCommit,
485 control: &StorageReadControl,
486 ) -> CommitResult {
487 self.check_transaction(transaction)?;
488 control.cancellation().check().map_err(VersionError::from)?;
489 let _bindings = write::reserve_bindings(prepared, control)?;
490 self.with_write(control, |connection| {
491 let _cache = commit_cache::CommitCache::grow(connection, prepared)?;
492 Ok(write::commit(
493 connection,
494 transaction,
495 prepared,
496 self.native,
497 control,
498 ))
499 })?
500 }
501 fn commit_status(
502 &self,
503 transaction: StorageTransactionId,
504 control: &StorageReadControl,
505 ) -> VersionResult<CommitStatus> {
506 self.check_transaction(transaction)?;
507 control.cancellation().check()?;
508 self.with(|connection| {
509 let read = connection.unchecked_transaction()?;
510 native::check_mapping(&read, self.native)?;
511 codec::header(&read, self.identity)?;
512 let status = codec::status(&read, transaction)?;
513 control.cancellation().check().map_err(VersionError::from)?;
514 read.commit()?;
515 Ok(status)
516 })
517 }
518
519 fn acknowledge_transaction(
520 &self,
521 acknowledgement: uqa_storage::mvcc::ReceiptAcknowledgement,
522 control: &StorageReadControl,
523 ) -> VersionResult<()> {
524 self.check_transaction(acknowledgement.transaction())?;
525 self.with_write(control, |connection| {
526 receipts::acknowledge(connection, self.native, acknowledgement, control)
527 })
528 }
529
530 fn reclaim_transaction_receipts(&self, control: &StorageReadControl) -> VersionResult<u64> {
531 self.reclaim_receipts(control)
532 }
533 fn abort(
534 &self,
535 transaction: StorageTransactionId,
536 control: &StorageReadControl,
537 ) -> VersionResult<CommitStatus> {
538 self.check_transaction(transaction)?;
539 control.cancellation().check()?;
540 self.with(|connection| write::abort(connection, transaction, self.native, control))
541 }
542}