1use std::path::Path;
9use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
10use std::sync::Arc;
11
12use rusqlite::OptionalExtension;
13
14use crate::database_owner_identity::{DatabaseOwnerIdentity, DatabaseOwnerIdentityError};
15use crate::error::SqliteError;
16use crate::pool::{ConnectionPool, PoolConfig, WalCeilingPolicy};
17use crate::stores::{agents, attachment, blob, entity, event, graph, note, sparse, text, vectors};
18
19mod capability_factories;
20mod code_map;
21mod core_stores;
22#[path = "backend/schema_readiness.rs"]
23mod memory_visibility;
24mod pack_schema;
25mod policy_open;
26
27#[cfg(test)]
28#[path = "backend/filter_builder_tests.rs"]
29mod filter_builder_tests;
30
31mod namespace_listing;
32pub use namespace_listing::NamespaceLiveness;
33
34#[cfg(test)]
35#[path = "backend/memory_visibility_tests.rs"]
36mod memory_visibility_tests;
37
38#[cfg(any(unix, windows))]
39mod claimed_file_identity;
40
41fn sqlite_table_exists(conn: &rusqlite::Connection, table: &str) -> Result<bool, SqliteError> {
42 conn.query_row(
43 "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1",
44 rusqlite::params![table],
45 |row| row.get::<_, i64>(0),
46 )
47 .optional()
48 .map(|row| row.is_some())
49 .map_err(SqliteError::Rusqlite)
50}
51
52fn ensure_fts_rowid_map_backfilled(
112 conn: &rusqlite::Connection,
113 table: &str,
114 admission: &crate::pool::WriteAdmission,
115) -> Result<bool, SqliteError> {
116 let map = text::rowid_map_table(table);
117 let state = text::rowid_map_state_table(table);
118 let already_backfilled: bool = conn.query_row(
119 &format!("SELECT EXISTS(SELECT 1 FROM {state} WHERE key = 'backfill' AND value = ?1)"),
120 rusqlite::params![text::ROWID_MAP_BACKFILL_COMPLETE],
121 |row| row.get(0),
122 )?;
123 if already_backfilled {
124 return Ok(true);
125 }
126 let fts_has_a_row: bool = conn.query_row(
127 &format!("SELECT EXISTS(SELECT 1 FROM {table} LIMIT 1)"),
128 [],
129 |row| row.get(0),
130 )?;
131 if !fts_has_a_row {
132 return Ok(false);
133 }
134
135 conn.execute_batch("BEGIN IMMEDIATE")?;
136 if let Err(error) = admission.check() {
137 return Err(crate::migrations::capacity_refusal_after_rollback(
138 conn,
139 conn.execute_batch("ROLLBACK"),
140 error,
141 "FTS rowid-map backfill",
142 ));
143 }
144 let result: Result<(), SqliteError> = (|| {
145 conn.execute_batch(&format!(
146 "DELETE FROM {map} WHERE NOT EXISTS ( \
147 SELECT 1 FROM {table} \
148 WHERE {table}.rowid = {map}.rowid \
149 AND {table}.namespace = {map}.namespace \
150 AND {table}.subject_id = {map}.subject_id \
151 )"
152 ))?;
153 conn.execute_batch(&format!(
154 "INSERT OR REPLACE INTO {map} (namespace, subject_id, rowid) \
155 SELECT namespace, subject_id, rowid FROM {table} \
156 WHERE namespace IS NOT NULL AND subject_id IS NOT NULL \
157 ORDER BY updated_at ASC, rowid ASC"
158 ))?;
159 conn.execute_batch(&format!(
160 "DELETE FROM {table} \
161 WHERE namespace IS NOT NULL AND subject_id IS NOT NULL \
162 AND rowid NOT IN (SELECT rowid FROM {map})"
163 ))?;
164 conn.execute(
165 &format!("INSERT OR REPLACE INTO {state} (key, value) VALUES ('backfill', ?1)"),
166 rusqlite::params![text::ROWID_MAP_BACKFILL_COMPLETE],
167 )?;
168 Ok(())
169 })();
170
171 match result {
172 Ok(()) => {
173 conn.execute_batch("COMMIT")?;
174 Ok(true)
175 }
176 Err(e) => {
177 let _ = conn.execute_batch("ROLLBACK");
178 Err(e)
179 }
180 }
181}
182
183fn warn_scan_fallback_once(table: &str) {
189 static WARNED: std::sync::Once = std::sync::Once::new();
190 WARNED.call_once(|| {
191 tracing::warn!(
192 table,
193 "opened a read-only text-search table with no rowid-map sidecar, or with a sidecar \
194 that has never proven a completed backfill (no durable completion marker); a \
195 read-only connection cannot create, backfill, or reconcile the map itself, so this \
196 falls back to pre-map namespace/subject_id scan predicates for get/delete on this \
197 table rather than trusting a map that might be partial"
198 );
199 });
200}
201
202fn validate_vector_model_key(model_key: &str) -> Result<(), SqliteError> {
203 if model_key.is_empty()
204 || !model_key
205 .chars()
206 .all(|c| c.is_ascii_alphanumeric() || c == '_')
207 {
208 return Err(SqliteError::InvalidData(format!(
209 "invalid model_key '{}': must be non-empty and contain only \
210 alphanumeric/underscore characters",
211 model_key
212 )));
213 }
214 Ok(())
215}
216
217fn validate_vector_table_columns(
218 conn: &rusqlite::Connection,
219 table: &str,
220) -> Result<(), SqliteError> {
221 let pragma = format!("PRAGMA table_xinfo({table})");
222 let mut stmt = conn.prepare(&pragma)?;
223 let mut rows = stmt.query([])?;
224 let mut has_field = false;
225 let mut has_embedding_model = false;
226 while let Some(row) = rows.next()? {
227 let name: String = row.get(1)?;
228 if name == "field" {
229 has_field = true;
230 }
231 if name == "embedding_model" {
232 has_embedding_model = true;
233 }
234 }
235 if !has_field || !has_embedding_model {
236 return Err(SqliteError::InvalidData(format!(
237 "vec0 table '{table}' is missing required column(s) (field={has_field}, \
238 embedding_model={has_embedding_model}); this is a pre-v0.2.8 vector schema and is \
239 not supported — recreate the database"
240 )));
241 }
242 Ok(())
243}
244
245#[derive(Clone, Copy, Debug)]
246pub(crate) enum StoreSchemaKind {
247 Entities,
248 Graph,
249 Notes,
250 Events,
251 Agents,
252}
253
254impl StoreSchemaKind {
255 fn initializer(self) -> fn(&rusqlite::Connection) -> Result<(), rusqlite::Error> {
256 match self {
257 Self::Entities => entity::ensure_entities_schema,
258 Self::Graph => graph::ensure_graph_schema,
259 Self::Notes => note::ensure_notes_schema,
260 Self::Events => event::ensure_events_schema,
261 Self::Agents => agents::ensure_agents_schema,
262 }
263 }
264}
265
266#[derive(Default)]
267pub(crate) struct StoreSchemaGate {
268 pub(crate) ready: AtomicBool,
269 #[cfg(test)]
270 pub(crate) attempts: AtomicUsize,
271}
272
273impl StoreSchemaGate {
274 pub(crate) fn ensure(
275 &self,
276 conn: &rusqlite::Connection,
277 ensure: fn(&rusqlite::Connection) -> Result<(), rusqlite::Error>,
278 ) -> Result<(), rusqlite::Error> {
279 if self.ready.load(Ordering::Acquire) {
282 return Ok(());
283 }
284 #[cfg(test)]
285 self.attempts.fetch_add(1, Ordering::Relaxed);
286 ensure(conn)?;
287 self.ready.store(true, Ordering::Release);
288 Ok(())
289 }
290}
291
292pub struct StorageBackend {
298 pool: Arc<ConnectionPool>,
299 is_file_backed: bool,
300 path: Option<std::path::PathBuf>,
301 vector_tables_ready: parking_lot::Mutex<std::collections::HashSet<String>>,
309 text_tables_ready: parking_lot::Mutex<std::collections::HashSet<String>>,
317 notes_seq_repair_runs: AtomicUsize,
324 store_schemas: [Arc<StoreSchemaGate>; 5],
325}
326
327impl StorageBackend {
328 pub fn sqlite(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
342 Self::sqlite_with_pool_config(path, PoolConfig::default(), None)
343 }
344
345 #[cfg(any(test, feature = "test-support"))]
347 pub fn sqlite_for_test(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
348 Self::sqlite_with_pool_config(path, PoolConfig::for_test(), None)
349 }
350
351 #[cfg(any(test, feature = "test-support"))]
353 pub fn sqlite_for_test_with_journal_mode(
354 path: impl AsRef<Path>,
355 wal_mode: bool,
356 busy_timeout: std::time::Duration,
357 ) -> Result<Self, SqliteError> {
358 Self::sqlite_with_pool_config(
359 path,
360 PoolConfig {
361 wal_mode,
362 busy_timeout,
363 write_queue_enabled: Some(true),
364 ..PoolConfig::for_test()
365 },
366 None,
367 )
368 }
369
370 #[cfg(any(test, feature = "test-support"))]
375 pub fn sqlite_for_test_with_journal_mode_in(
376 path: impl AsRef<Path>,
377 wal_mode: bool,
378 busy_timeout: std::time::Duration,
379 volume_lock_dir: std::path::PathBuf,
380 ) -> Result<Self, SqliteError> {
381 Self::sqlite_with_pool_config(
382 path,
383 PoolConfig {
384 wal_mode,
385 busy_timeout,
386 write_queue_enabled: Some(true),
387 volume_lock_dir: Some(volume_lock_dir),
388 ..PoolConfig::for_test()
389 },
390 None,
391 )
392 }
393
394 pub fn sqlite_with_max_readers(
398 path: impl AsRef<Path>,
399 max_readers: Option<usize>,
400 ) -> Result<Self, SqliteError> {
401 Self::sqlite_with_pool_config(path, PoolConfig::default(), max_readers)
402 }
403
404 pub fn sqlite_with_max_readers_and_wal_ceiling(
408 path: impl AsRef<Path>,
409 max_readers: Option<usize>,
410 wal_ceiling: WalCeilingPolicy,
411 ) -> Result<Self, SqliteError> {
412 Self::sqlite_with_pool_config(
413 path,
414 PoolConfig {
415 wal_ceiling,
416 ..PoolConfig::default()
417 },
418 max_readers,
419 )
420 }
421
422 pub(crate) fn sqlite_with_pool_config(
423 path: impl AsRef<Path>,
424 pool_config: PoolConfig,
425 max_readers: Option<usize>,
426 ) -> Result<Self, SqliteError> {
427 crate::extension::ensure_extensions_loaded();
428 let resolved = path.as_ref().to_path_buf();
429 let read_only =
430 std::fs::metadata(&resolved).is_ok_and(|metadata| metadata.permissions().readonly());
431 let mut config = PoolConfig {
432 path: Some(resolved.clone()),
433 read_only,
434 ..pool_config
435 };
436 if let Some(max_readers) = max_readers {
437 config.max_readers = max_readers;
438 }
439 if read_only {
440 config.write_queue_enabled = Some(false);
441 }
442 let pool = ConnectionPool::new(config)?;
443 Ok(Self {
444 pool: Arc::new(pool),
445 is_file_backed: true,
446 path: Some(resolved),
447 vector_tables_ready: Default::default(),
448 text_tables_ready: Default::default(),
449 notes_seq_repair_runs: AtomicUsize::new(0),
450 store_schemas: std::array::from_fn(|_| Arc::new(StoreSchemaGate::default())),
451 })
452 }
453
454 pub fn sqlite_read_only(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
465 Self::sqlite_read_only_with_pool_config(path, PoolConfig::default(), None)
466 }
467
468 #[cfg(any(test, feature = "test-support"))]
470 pub fn sqlite_read_only_for_test(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
471 Self::sqlite_read_only_with_pool_config(path, PoolConfig::for_test(), None)
472 }
473
474 pub fn sqlite_read_only_with_max_readers(
476 path: impl AsRef<Path>,
477 max_readers: Option<usize>,
478 ) -> Result<Self, SqliteError> {
479 Self::sqlite_read_only_with_pool_config(path, PoolConfig::default(), max_readers)
480 }
481
482 pub fn sqlite_read_only_with_max_readers_and_wal_ceiling(
485 path: impl AsRef<Path>,
486 max_readers: Option<usize>,
487 wal_ceiling: WalCeilingPolicy,
488 ) -> Result<Self, SqliteError> {
489 Self::sqlite_read_only_with_pool_config(
490 path,
491 PoolConfig {
492 wal_ceiling,
493 ..PoolConfig::default()
494 },
495 max_readers,
496 )
497 }
498
499 pub fn sqlite_read_only_live_with_max_readers_and_wal_ceiling(
504 path: impl AsRef<Path>,
505 max_readers: Option<usize>,
506 wal_ceiling: WalCeilingPolicy,
507 ) -> Result<Self, SqliteError> {
508 Self::sqlite_read_only_with_pool_constructor(
509 path,
510 PoolConfig {
511 wal_ceiling,
512 ..PoolConfig::default()
513 },
514 max_readers,
515 ConnectionPool::new_read_only_live,
516 )
517 }
518
519 fn sqlite_read_only_with_pool_config(
520 path: impl AsRef<Path>,
521 pool_config: PoolConfig,
522 max_readers: Option<usize>,
523 ) -> Result<Self, SqliteError> {
524 Self::sqlite_read_only_with_pool_constructor(
525 path,
526 pool_config,
527 max_readers,
528 ConnectionPool::new,
529 )
530 }
531
532 fn sqlite_read_only_with_pool_constructor(
533 path: impl AsRef<Path>,
534 pool_config: PoolConfig,
535 max_readers: Option<usize>,
536 open_pool: fn(PoolConfig) -> Result<ConnectionPool, SqliteError>,
537 ) -> Result<Self, SqliteError> {
538 crate::extension::ensure_extensions_loaded();
539 let resolved = path.as_ref().to_path_buf();
540 let mut config = PoolConfig {
541 path: Some(resolved.clone()),
542 read_only: true,
543 write_queue_enabled: Some(false),
544 ..pool_config
545 };
546 if let Some(max_readers) = max_readers {
547 config.max_readers = max_readers;
548 }
549 let pool = open_pool(config)?;
555 Ok(Self {
556 pool: Arc::new(pool),
557 is_file_backed: true,
558 path: Some(resolved),
559 vector_tables_ready: Default::default(),
560 text_tables_ready: Default::default(),
561 notes_seq_repair_runs: AtomicUsize::new(0),
562 store_schemas: std::array::from_fn(|_| Arc::new(StoreSchemaGate::default())),
563 })
564 }
565
566 pub fn memory() -> Result<Self, SqliteError> {
572 crate::extension::ensure_extensions_loaded();
573 let config = PoolConfig {
574 path: None,
575 ..PoolConfig::default()
576 };
577 let pool = ConnectionPool::new(config)?;
578 Ok(Self {
579 pool: Arc::new(pool),
580 is_file_backed: false,
581 path: None,
582 vector_tables_ready: Default::default(),
583 text_tables_ready: Default::default(),
584 notes_seq_repair_runs: AtomicUsize::new(0),
585 store_schemas: std::array::from_fn(|_| Arc::new(StoreSchemaGate::default())),
586 })
587 }
588
589 pub fn apply_schema(
596 &self,
597 plan: &crate::migrations::ServiceSchemaPlan,
598 ) -> Result<(), SqliteError> {
599 let admission = self.pool.write_admission();
600 crate::migrations::apply_schema_plan_with_admission(
601 &mut self.pool.migration_transactions(),
602 plan,
603 &admission,
604 )
605 }
606
607 pub fn apply_pack_ddl_statements(
623 &self,
624 statements: &[&'static str],
625 ) -> Result<(), SqliteError> {
626 self.apply_pack_ddl_statements_with_columns(statements, &[])
627 }
628
629 pub fn apply_pack_ddl_statements_with_columns(
636 &self,
637 statements: &[&'static str],
638 additions: &[khive_types::PackColumnAddition],
639 ) -> Result<(), SqliteError> {
640 let writer = self.pool.writer()?;
641 writer.transaction(|conn| {
642 pack_schema::add_missing_columns(conn, additions)?;
643 for &stmt in statements {
644 conn.execute_batch(stmt)?;
645 }
646 pack_schema::validate_columns(conn, additions)?;
647 Ok(())
648 })
649 }
650
651 pub fn validate_pack_schema_columns(
654 &self,
655 additions: &[khive_types::PackColumnAddition],
656 ) -> Result<(), SqliteError> {
657 if additions.is_empty() {
658 return Ok(());
659 }
660 let reader = self.pool.reader()?;
661 pack_schema::validate_columns(reader.conn(), additions)
662 }
663
664 pub fn prepare_core_schema(&self) -> Result<u32, SqliteError> {
674 if self.is_read_only() {
675 let reader = self.pool.reader()?;
676 crate::migrations::validate_schema_is_current(reader.conn())
677 } else {
678 let latest = crate::migrations::MIGRATIONS
679 .last()
680 .map(|migration| migration.version)
681 .unwrap_or(0);
682 {
683 let reader = self.pool.reader()?;
684 let current = crate::migrations::read_schema_version(reader.conn())?;
685 if current >= latest {
686 return crate::migrations::validate_schema_is_current(reader.conn());
687 }
688 }
689 let owner = crate::stores::blob::acquire_database_gc_owner_for_path_blocking(
690 self.pool.canonical_path().map(Path::to_path_buf),
691 )
692 .map_err(|error| {
693 SqliteError::InvalidData(format!(
694 "failed to acquire database GC owner before schema preparation: {error}"
695 ))
696 })?;
697 self.run_core_migrations(&owner)
698 }
699 }
700
701 pub fn attachment_cutover_status(
703 &self,
704 ) -> Result<crate::migrations::AttachmentCutoverStatus, SqliteError> {
705 let reader = self.pool.reader()?;
706 crate::migrations::attachment_cutover_status(reader.conn())
707 }
708
709 fn require_attachment_cutover_owner(
710 &self,
711 owner: &crate::stores::blob::DatabaseGcOwnerGuard,
712 ) -> Result<(), SqliteError> {
713 let sql = self.sql();
714 let backend_path = sql.database_path();
715 if owner.database_path() != backend_path.as_deref() {
716 return Err(SqliteError::InvalidData(format!(
717 "attachment cutover GC owner targets {:?}, but this backend is {:?}",
718 owner.database_path(),
719 backend_path.as_deref()
720 )));
721 }
722 Ok(())
723 }
724
725 pub fn stage_attachment_cutover(
729 &self,
730 owner: &crate::stores::blob::DatabaseGcOwnerGuard,
731 ) -> Result<(), SqliteError> {
732 self.require_attachment_cutover_owner(owner)?;
733 if self.is_read_only() {
734 return Err(SqliteError::InvalidData(
735 "cannot stage attachment cutover on a read-only backend".into(),
736 ));
737 }
738 let mut writer = self.pool.writer()?;
739 let admission = self.pool.write_admission();
740 crate::migrations::stage_attachment_cutover_with_admission(writer.conn_mut(), &admission)
741 }
742
743 pub fn apply_verified_attachments(
745 &self,
746 owner: &crate::stores::blob::DatabaseGcOwnerGuard,
747 attachments: &[khive_storage::Attachment],
748 ) -> Result<(), SqliteError> {
749 self.require_attachment_cutover_owner(owner)?;
750 if self.is_read_only() {
751 return Err(SqliteError::InvalidData(
752 "cannot apply verified attachments on a read-only backend".into(),
753 ));
754 }
755 let mut writer = self.pool.writer()?;
756 let tx = writer
757 .conn_mut()
758 .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
759 let admission = self.pool.write_admission();
760 if let Err(error) = admission.check() {
761 let rollback = tx.rollback();
762 return Err(crate::migrations::capacity_refusal_after_rollback(
763 writer.conn(),
764 rollback,
765 error,
766 "verified attachment publication",
767 ));
768 }
769 for attachment in attachments {
770 attachment
771 .validate()
772 .map_err(|error| SqliteError::InvalidData(error.to_string()))?;
773 crate::migrations::apply_generic_verified_attachment(
774 &tx,
775 &attachment.record_uuid.to_string(),
776 attachment.substrate.as_str(),
777 &attachment.role,
778 &attachment.content_ref,
779 attachment.media_type.as_deref(),
780 attachment.size_bytes,
781 attachment.created_at,
782 )?;
783 }
784 tx.commit()?;
785 Ok(())
786 }
787
788 pub fn finalize_attachment_cutover(
791 &self,
792 owner: &crate::stores::blob::DatabaseGcOwnerGuard,
793 ) -> Result<(), SqliteError> {
794 self.require_attachment_cutover_owner(owner)?;
795 if self.is_read_only() {
796 return Err(SqliteError::InvalidData(
797 "cannot finalize attachment cutover on a read-only backend".into(),
798 ));
799 }
800 let mut writer = self.pool.writer()?;
801 let admission = self.pool.write_admission();
802 crate::migrations::finalize_attachment_cutover_with_admission(writer.conn_mut(), &admission)
803 }
804
805 pub fn entities(&self) -> Result<Arc<dyn khive_storage::EntityStore>, SqliteError> {
809 self.entities_for_namespace("local")
810 }
811
812 pub fn entities_for_namespace(
816 &self,
817 namespace: &str,
818 ) -> Result<Arc<dyn khive_storage::EntityStore>, SqliteError> {
819 if namespace.trim().is_empty() {
820 return Err(SqliteError::InvalidData(
821 "entities namespace must be non-empty".to_string(),
822 ));
823 }
824 self.ensure_store_schema(StoreSchemaKind::Entities)?;
825
826 Ok(Arc::new(entity::SqlEntityStore::new(
827 Arc::clone(&self.pool),
828 self.is_file_backed,
829 )))
830 }
831
832 pub fn attachments(&self) -> Result<Arc<dyn khive_storage::AttachmentStore>, SqliteError> {
839 Ok(Arc::new(attachment::SqlAttachmentStore::new(
840 Arc::clone(&self.pool),
841 self.is_file_backed,
842 )))
843 }
844
845 pub fn graph(&self) -> Result<Arc<dyn khive_storage::GraphStore>, SqliteError> {
850 self.graph_for_namespace("local")
851 }
852
853 pub fn graph_for_namespace(
855 &self,
856 namespace: &str,
857 ) -> Result<Arc<dyn khive_storage::GraphStore>, SqliteError> {
858 if namespace.trim().is_empty() {
859 return Err(SqliteError::InvalidData(
860 "graph namespace must be non-empty".to_string(),
861 ));
862 }
863 self.ensure_store_schema(StoreSchemaKind::Graph)?;
864
865 Ok(Arc::new(
866 graph::SqlGraphStore::new_scoped(
867 Arc::clone(&self.pool),
868 self.is_file_backed,
869 namespace.trim().to_string(),
870 )
871 .with_index_repair(crate::stores::index_repair::IndexRepairContext::new(
872 Arc::clone(&self.pool),
873 self.store_schemas.clone(),
874 crate::stores::index_repair::IndexReadKind::Graph,
875 )),
876 ))
877 }
878
879 fn ensure_store_schema(&self, kind: StoreSchemaKind) -> Result<(), SqliteError> {
880 if self.is_read_only()
881 || self.store_schemas[kind as usize]
882 .ready
883 .load(Ordering::Acquire)
884 {
885 return Ok(());
886 }
887 let writer = self.constructor_writer()?;
888 self.ensure_store_schema_with_writer(kind, writer.conn())
889 }
890
891 fn ensure_store_schema_with_writer(
892 &self,
893 kind: StoreSchemaKind,
894 conn: &rusqlite::Connection,
895 ) -> Result<(), SqliteError> {
896 self.store_schemas[kind as usize].ensure(conn, kind.initializer())?;
897 Ok(())
898 }
899
900 fn constructor_writer(
901 &self,
902 ) -> Result<crate::pool::PooledAutocommitWriteUnit<'_>, SqliteError> {
903 let context = khive_storage::capture_request_read_context();
904 let Some(operation) = context.store_acquisition_operation() else {
905 return self.pool.autocommit_write_unit();
906 };
907 let writer = self
908 .pool
909 .writer_until_for_admitted_operation(|| context.blocking_stop_reason().is_some())?
910 .ok_or_else(|| {
911 SqliteError::RequestReadStopped(khive_storage::StorageError::Timeout {
912 operation: operation.into(),
913 })
914 })?;
915 writer.admit_autocommit()
916 }
917
918 pub fn notes(&self) -> Result<Arc<dyn khive_storage::NoteStore>, SqliteError> {
922 self.notes_for_namespace("local")
923 }
924
925 pub fn notes_for_namespace(
929 &self,
930 namespace: &str,
931 ) -> Result<Arc<dyn khive_storage::NoteStore>, SqliteError> {
932 if namespace.trim().is_empty() {
933 return Err(SqliteError::InvalidData(
934 "notes namespace must be non-empty".to_string(),
935 ));
936 }
937 if !self.is_read_only()
938 && (!self.store_schemas[StoreSchemaKind::Notes as usize]
939 .ready
940 .load(Ordering::Acquire)
941 || self.notes_seq_repair_runs.load(Ordering::Relaxed) == 0)
942 {
943 let writer = self.constructor_writer()?;
944 self.ensure_store_schema_with_writer(StoreSchemaKind::Notes, writer.conn())?;
945
946 if self.notes_seq_repair_runs.load(Ordering::Relaxed) == 0 {
953 note::repair_notes_seq(writer.conn())?;
954 self.notes_seq_repair_runs.fetch_add(1, Ordering::Relaxed);
955 }
956 }
957
958 Ok(Arc::new(
959 note::SqlNoteStore::new(Arc::clone(&self.pool), self.is_file_backed).with_index_repair(
960 crate::stores::index_repair::IndexRepairContext::new(
961 Arc::clone(&self.pool),
962 self.store_schemas.clone(),
963 crate::stores::index_repair::IndexReadKind::Notes,
964 ),
965 ),
966 ))
967 }
968
969 pub fn notes_seq_repair_run_count(&self) -> usize {
974 self.notes_seq_repair_runs.load(Ordering::Relaxed)
975 }
976
977 pub fn events(&self) -> Result<Arc<dyn khive_storage::EventStore>, SqliteError> {
982 self.events_for_namespace("local")
983 }
984
985 pub fn events_for_namespace(
987 &self,
988 namespace: &str,
989 ) -> Result<Arc<dyn khive_storage::EventStore>, SqliteError> {
990 if namespace.trim().is_empty() {
991 return Err(SqliteError::InvalidData(
992 "events namespace must be non-empty".to_string(),
993 ));
994 }
995 self.ensure_store_schema(StoreSchemaKind::Events)?;
996
997 Ok(Arc::new(event::SqlEventStore::new_scoped(
998 Arc::clone(&self.pool),
999 self.is_file_backed,
1000 namespace.trim().to_string(),
1001 )))
1002 }
1003
1004 pub fn agents(&self) -> Result<Arc<dyn khive_storage::AgentStore>, SqliteError> {
1009 self.ensure_store_schema(StoreSchemaKind::Agents)?;
1010
1011 Ok(Arc::new(agents::SqlAgentStore::new(
1012 Arc::clone(&self.pool),
1013 self.is_file_backed,
1014 )))
1015 }
1016
1017 pub fn vectors(
1023 &self,
1024 model_key: &str,
1025 embedding_model: &str,
1026 dimensions: usize,
1027 ) -> Result<Arc<dyn khive_storage::VectorStore>, SqliteError> {
1028 self.vectors_for_namespace(model_key, embedding_model, dimensions, "local")
1029 }
1030
1031 pub fn vectors_for_namespace(
1041 &self,
1042 model_key: &str,
1043 embedding_model: &str,
1044 dimensions: usize,
1045 namespace: &str,
1046 ) -> Result<Arc<dyn khive_storage::VectorStore>, SqliteError> {
1047 validate_vector_model_key(model_key)?;
1048 if namespace.trim().is_empty() {
1049 return Err(SqliteError::InvalidData(
1050 "vector store namespace must be non-empty".to_string(),
1051 ));
1052 }
1053 self.ensure_vector_tables(&[(model_key, dimensions)])?;
1054 Ok(Arc::new(vectors::SqliteVecStore::new(
1055 Arc::clone(&self.pool),
1056 self.is_file_backed,
1057 model_key.to_string(),
1058 embedding_model.to_string(),
1059 dimensions,
1060 namespace.trim().to_string(),
1061 )?))
1062 }
1063
1064 pub fn ensure_vector_tables(&self, models: &[(&str, usize)]) -> Result<(), SqliteError> {
1073 for (model_key, _) in models {
1074 validate_vector_model_key(model_key)?;
1075 }
1076 if models.is_empty() {
1077 return Ok(());
1078 }
1079
1080 crate::extension::ensure_extensions_loaded();
1082
1083 if self.is_read_only() {
1088 let reader = self.pool.reader()?;
1092 for (model_key, _) in models {
1093 let table = format!("vec_{model_key}");
1094 if !sqlite_table_exists(reader.conn(), &table)? {
1095 return Err(SqliteError::InvalidData(format!(
1096 "read-only database has no vector table '{table}'; create and populate it in \
1097 a writable copy before opening the snapshot"
1098 )));
1099 }
1100 validate_vector_table_columns(reader.conn(), &table)?;
1101 }
1102 return Ok(());
1103 }
1104
1105 let pending = self.unprepared_vector_tables(models);
1106 if pending.is_empty() {
1107 return Ok(());
1108 }
1109 let writer = self.constructor_writer()?;
1110 let pending = self.unprepared_vector_tables(&pending);
1113 if pending.is_empty() {
1114 return Ok(());
1115 }
1116
1117 for (model_key, _) in &pending {
1121 let table = format!("vec_{model_key}");
1122 if sqlite_table_exists(writer.conn(), &table)? {
1128 validate_vector_table_columns(writer.conn(), &table)?;
1129 }
1130 }
1131
1132 writer
1141 .conn()
1142 .execute_batch(crate::migrations::EMBEDDING_MODELS_DDL)?;
1143
1144 writer
1148 .conn()
1149 .execute_batch(crate::migrations::ANN_WRITE_LOG_DDL)?;
1150 writer
1151 .conn()
1152 .execute_batch(crate::migrations::ANN_WRITE_LOG_MODEL_SEQ_INDEX_DDL)?;
1153 writer
1154 .conn()
1155 .execute_batch(crate::migrations::ANN_CONSUMER_PENDING_DDL)?;
1156 for (model_key, dimensions) in &pending {
1158 let ddl = format!(
1159 "CREATE VIRTUAL TABLE IF NOT EXISTS vec_{} USING vec0(\
1160 subject_id TEXT PRIMARY KEY, \
1161 namespace TEXT NOT NULL, \
1162 kind TEXT NOT NULL, \
1163 field TEXT NOT NULL, \
1164 embedding_model TEXT NOT NULL, \
1165 embedding float[{}] distance_metric=cosine\
1166 )",
1167 model_key, dimensions
1168 );
1169 writer.conn().execute_batch(&ddl)?;
1170 }
1171 self.vector_tables_ready.lock().extend(
1174 pending
1175 .iter()
1176 .map(|(model_key, _)| (*model_key).to_string()),
1177 );
1178 Ok(())
1179 }
1180
1181 fn unprepared_vector_tables<'a>(&self, models: &[(&'a str, usize)]) -> Vec<(&'a str, usize)> {
1183 let ready = self.vector_tables_ready.lock();
1184 models
1185 .iter()
1186 .filter(|(model_key, _)| !ready.contains(*model_key))
1187 .copied()
1188 .collect()
1189 }
1190
1191 pub fn register_embedding_model(
1196 &self,
1197 engine_name: &str,
1198 model_id: &str,
1199 key_version: &str,
1200 dimensions: u32,
1201 ) -> Result<(), SqliteError> {
1202 let writer = self.pool.autocommit_write_unit()?;
1203 writer
1204 .conn()
1205 .execute_batch(crate::migrations::EMBEDDING_MODELS_DDL)
1206 .map_err(|error| {
1207 SqliteError::write(error, khive_storage::error::SqliteWriteStage::Statement)
1208 })?;
1209
1210 let now = chrono::Utc::now().timestamp_micros();
1211 let canonical_key =
1212 format!("{engine_name}:{model_id}:{key_version}:{dimensions}").into_bytes();
1213 let id = uuid::Uuid::new_v4();
1214 writer
1215 .conn()
1216 .execute(
1217 "INSERT INTO _embedding_models \
1218 (id, engine_name, model_id, key_version, dim, output_dim, status, \
1219 activated_at, superseded_at, superseded_by, canonical_key, created_at) \
1220 VALUES (?1, ?2, ?3, ?4, ?5, NULL, 'active', ?6, NULL, NULL, ?7, ?8) \
1221 ON CONFLICT(canonical_key) DO UPDATE SET \
1222 status = 'active', \
1223 activated_at = COALESCE(_embedding_models.activated_at, excluded.activated_at)",
1224 rusqlite::params![
1225 id.as_bytes().as_slice(),
1226 engine_name,
1227 model_id,
1228 key_version,
1229 dimensions as i64,
1230 now,
1231 canonical_key,
1232 now,
1233 ],
1234 )
1235 .map_err(|error| {
1236 SqliteError::write(error, khive_storage::error::SqliteWriteStage::Statement)
1237 })?;
1238 Ok(())
1239 }
1240
1241 pub fn sparse(
1245 &self,
1246 model_key: &str,
1247 ) -> Result<Arc<dyn khive_storage::SparseStore>, SqliteError> {
1248 self.sparse_for_namespace(model_key, "local")
1249 }
1250
1251 pub fn sparse_for_namespace(
1255 &self,
1256 model_key: &str,
1257 namespace: &str,
1258 ) -> Result<Arc<dyn khive_storage::SparseStore>, SqliteError> {
1259 if model_key.is_empty()
1260 || !model_key
1261 .chars()
1262 .all(|c| c.is_ascii_alphanumeric() || c == '_')
1263 {
1264 return Err(SqliteError::InvalidData(format!(
1265 "invalid model_key '{}': must be non-empty and contain only alphanumeric/underscore characters",
1266 model_key
1267 )));
1268 }
1269 if namespace.trim().is_empty() {
1270 return Err(SqliteError::InvalidData(
1271 "sparse store namespace must be non-empty".to_string(),
1272 ));
1273 }
1274
1275 if self.is_read_only() {
1276 let table = format!("sparse_{model_key}");
1277 let reader = self.pool.reader()?;
1278 if !sqlite_table_exists(reader.conn(), &table)? {
1279 return Err(SqliteError::InvalidData(format!(
1280 "read-only database has no sparse table '{table}'; create and populate it in \
1281 a writable copy before opening the snapshot"
1282 )));
1283 }
1284 } else {
1285 let writer = self.pool.autocommit_write_unit()?;
1286 sparse::ensure_sparse_schema(writer.conn(), model_key)
1287 .map_err(SqliteError::Rusqlite)?;
1288 }
1289
1290 Ok(Arc::new(sparse::SqliteSparseStore::new(
1291 Arc::clone(&self.pool),
1292 self.is_file_backed,
1293 model_key.to_string(),
1294 namespace.trim().to_string(),
1295 )?))
1296 }
1297
1298 pub fn text(&self, table_key: &str) -> Result<Arc<dyn khive_storage::TextSearch>, SqliteError> {
1305 self.text_with_tokenizer(table_key, "trigram")
1306 }
1307
1308 pub fn text_with_tokenizer(
1316 &self,
1317 table_key: &str,
1318 tokenizer: &str,
1319 ) -> Result<Arc<dyn khive_storage::TextSearch>, SqliteError> {
1320 if table_key.is_empty()
1321 || !table_key
1322 .chars()
1323 .all(|c| c.is_ascii_alphanumeric() || c == '_')
1324 {
1325 return Err(SqliteError::InvalidData(format!(
1326 "invalid table_key '{}': must be non-empty and contain only \
1327 alphanumeric/underscore characters",
1328 table_key
1329 )));
1330 }
1331 if table_key.ends_with("_rowids") || table_key.ends_with("_rowids_state") {
1341 return Err(SqliteError::InvalidData(format!(
1342 "invalid table_key '{}': must not end in '_rowids' or '_rowids_state' — those \
1343 suffixes are reserved for a text table's own rowid-map sidecar and its \
1344 completion-marker state table (see text::rowid_map_table, \
1345 text::rowid_map_state_table)",
1346 table_key
1347 )));
1348 }
1349 if tokenizer.is_empty()
1350 || !tokenizer
1351 .chars()
1352 .all(|c| c.is_ascii_alphanumeric() || c == '_')
1353 {
1354 return Err(SqliteError::InvalidData(format!(
1355 "invalid tokenizer '{}': must be non-empty and contain only \
1356 alphanumeric/underscore characters",
1357 tokenizer
1358 )));
1359 }
1360
1361 let ddl = format!(
1362 "CREATE VIRTUAL TABLE IF NOT EXISTS fts_{} USING fts5(\
1363 subject_id UNINDEXED, \
1364 kind UNINDEXED, \
1365 title, \
1366 body, \
1367 tags UNINDEXED, \
1368 namespace UNINDEXED, \
1369 metadata UNINDEXED, \
1370 updated_at UNINDEXED, \
1371 record_kind, \
1372 tokenize = '{}'\
1373 )",
1374 table_key, tokenizer
1375 );
1376 let table = format!("fts_{table_key}");
1377 if self.is_read_only() {
1378 let reader = self.pool.reader()?;
1379 if !sqlite_table_exists(reader.conn(), &table)? {
1380 return Err(SqliteError::InvalidData(format!(
1381 "read-only database has no text-search table '{table}'; create and populate \
1382 it in a writable copy before opening the snapshot"
1383 )));
1384 }
1385 let map = text::rowid_map_table(&table);
1399 let map_exists = sqlite_table_exists(reader.conn(), &map)?;
1400 let marker_present = if map_exists {
1401 let state = text::rowid_map_state_table(&table);
1402 sqlite_table_exists(reader.conn(), &state)?
1403 && reader.conn().query_row(
1404 &format!(
1405 "SELECT EXISTS(SELECT 1 FROM {state} WHERE key = 'backfill' AND value = ?1)"
1406 ),
1407 rusqlite::params![text::ROWID_MAP_BACKFILL_COMPLETE],
1408 |row| row.get(0),
1409 )?
1410 } else {
1411 false
1412 };
1413 if !map_exists || !marker_present {
1414 warn_scan_fallback_once(&table);
1415 return Ok(Arc::new(text::Fts5TextSearch::new_scan_fallback(
1416 Arc::clone(&self.pool),
1417 self.is_file_backed,
1418 table_key.to_string(),
1419 )));
1420 }
1421 } else if !self.text_tables_ready.lock().contains(&table) {
1422 let writer = self.pool.autocommit_write_unit()?;
1423 if !self.text_tables_ready.lock().contains(&table) {
1426 writer.conn().execute_batch(&ddl)?;
1427 writer.conn().execute_batch(&text::rowid_map_ddl(&table))?;
1428 let marker_durable = ensure_fts_rowid_map_backfilled(
1429 writer.conn(),
1430 &table,
1431 &self.pool.write_admission(),
1432 )?;
1433 if marker_durable {
1436 self.text_tables_ready.lock().insert(table.clone());
1437 }
1438 }
1439 }
1440
1441 Ok(Arc::new(text::Fts5TextSearch::new(
1442 Arc::clone(&self.pool),
1443 self.is_file_backed,
1444 table_key.to_string(),
1445 )))
1446 }
1447
1448 pub fn blob_store(
1457 &self,
1458 config_root: Option<&Path>,
1459 floor_bytes: Option<u64>,
1460 ) -> Result<Arc<dyn khive_storage::BlobStore>, SqliteError> {
1461 Ok(Arc::new(self.fs_blob_store(config_root, floor_bytes)?))
1462 }
1463
1464 pub(crate) fn fs_blob_store(
1467 &self,
1468 config_root: Option<&Path>,
1469 floor_bytes: Option<u64>,
1470 ) -> Result<blob::FsBlobStore, SqliteError> {
1471 let root = blob::resolve_blob_root(self.data_dir().as_deref(), config_root)?;
1472 let floor = blob::FsBlobStore::resolve_floor(&root, floor_bytes);
1473 blob::FsBlobStore::new(root, floor)
1474 }
1475
1476 pub fn blob_store_read_only(
1481 &self,
1482 config_root: Option<&Path>,
1483 floor_bytes: Option<u64>,
1484 ) -> Result<Arc<dyn khive_storage::BlobStore>, SqliteError> {
1485 let root = blob::resolve_blob_root(self.data_dir().as_deref(), config_root)?;
1486 let floor = blob::FsBlobStore::resolve_floor(&root, floor_bytes);
1487 Ok(Arc::new(blob::FsBlobStore::open_existing(root, floor)?))
1488 }
1489
1490 pub fn is_file_backed(&self) -> bool {
1492 self.is_file_backed
1493 }
1494
1495 pub fn is_read_only(&self) -> bool {
1498 self.pool.config().read_only
1499 }
1500
1501 pub fn data_dir(&self) -> Option<std::path::PathBuf> {
1504 self.path.as_ref()?.parent().map(|p| p.to_path_buf())
1505 }
1506
1507 pub fn ann_root(&self) -> Option<std::path::PathBuf> {
1515 ann_root_for(self.path.as_ref()?)
1516 }
1517
1518 pub fn pool(&self) -> &ConnectionPool {
1520 &self.pool
1521 }
1522
1523 pub fn database_owner_identity(
1526 &self,
1527 ) -> Result<DatabaseOwnerIdentity, DatabaseOwnerIdentityError> {
1528 self.pool.database_owner_identity()
1529 }
1530
1531 pub fn verify_database_owner(
1533 &self,
1534 expected: &DatabaseOwnerIdentity,
1535 ) -> Result<(), DatabaseOwnerIdentityError> {
1536 self.pool.verify_database_owner(expected)
1537 }
1538
1539 pub fn pool_arc(&self) -> Arc<ConnectionPool> {
1541 Arc::clone(&self.pool)
1542 }
1543}
1544
1545fn ann_root_for(path: &std::path::Path) -> Option<std::path::PathBuf> {
1550 let mut file = path.file_name()?.to_os_string();
1551 file.push(".ann");
1552 path.parent().map(|p| p.join(file))
1553}
1554
1555#[cfg(test)]
1556#[path = "backend/store_accessor_tests.rs"]
1557mod store_accessor_tests;
1558
1559#[cfg(test)]
1560#[path = "backend/store_accessor_index_tests.rs"]
1561mod store_accessor_index_tests;
1562
1563#[cfg(test)]
1564#[path = "backend_tests.rs"]
1565mod tests;
1566
1567#[cfg(test)]
1568#[path = "backend_admission_tests.rs"]
1569mod admission_tests;
1570
1571#[cfg(test)]
1572mod live_readonly_tests;