1use std::path::Path;
8use std::sync::atomic::{AtomicUsize, Ordering};
9use std::sync::Arc;
10
11use rusqlite::OptionalExtension;
12
13use crate::error::SqliteError;
14use crate::pool::{ConnectionPool, PoolConfig};
15use crate::sql_bridge::SqlBridge;
16use crate::stores::{blob, entity, event, graph, note, sparse, text, vectors};
17
18pub struct StorageBackend {
20 pool: Arc<ConnectionPool>,
21 is_file_backed: bool,
22 path: Option<std::path::PathBuf>,
23 notes_seq_repair_runs: AtomicUsize,
30}
31
32impl StorageBackend {
33 pub fn sqlite(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
39 crate::extension::ensure_extensions_loaded();
40 let resolved = path.as_ref().to_path_buf();
41 let config = PoolConfig {
42 path: Some(resolved.clone()),
43 ..PoolConfig::default()
44 };
45 let pool = ConnectionPool::new(config)?;
46 Ok(Self {
47 pool: Arc::new(pool),
48 is_file_backed: true,
49 path: Some(resolved),
50 notes_seq_repair_runs: AtomicUsize::new(0),
51 })
52 }
53
54 pub fn sqlite_read_only(path: impl AsRef<Path>) -> Result<Self, SqliteError> {
64 crate::extension::ensure_extensions_loaded();
65 let resolved = path.as_ref().to_path_buf();
66 let config = PoolConfig {
67 path: Some(resolved.clone()),
68 read_only: true,
69 ..PoolConfig::default()
70 };
71 let pool = ConnectionPool::new(config)?;
77 Ok(Self {
78 pool: Arc::new(pool),
79 is_file_backed: true,
80 path: Some(resolved),
81 notes_seq_repair_runs: AtomicUsize::new(0),
82 })
83 }
84
85 pub fn memory() -> Result<Self, SqliteError> {
91 crate::extension::ensure_extensions_loaded();
92 let config = PoolConfig {
93 path: None,
94 ..PoolConfig::default()
95 };
96 let pool = ConnectionPool::new(config)?;
97 Ok(Self {
98 pool: Arc::new(pool),
99 is_file_backed: false,
100 path: None,
101 notes_seq_repair_runs: AtomicUsize::new(0),
102 })
103 }
104
105 pub fn sql(&self) -> Arc<dyn khive_storage::SqlAccess> {
109 Arc::new(SqlBridge::new(Arc::clone(&self.pool), self.is_file_backed))
110 }
111
112 pub fn apply_schema(
118 &self,
119 plan: &crate::migrations::ServiceSchemaPlan,
120 ) -> Result<(), SqliteError> {
121 let writer = self.pool.try_writer()?;
122 crate::migrations::apply_schema_plan(writer.conn(), plan)
123 }
124
125 pub fn apply_pack_ddl_statements(
142 &self,
143 statements: &[&'static str],
144 ) -> Result<(), SqliteError> {
145 let writer = self.pool.try_writer()?;
146 for &stmt in statements {
147 writer.conn().execute_batch(stmt)?;
148 }
149 Ok(())
150 }
151
152 pub fn entities(&self) -> Result<Arc<dyn khive_storage::EntityStore>, SqliteError> {
156 self.entities_for_namespace("local")
157 }
158
159 pub fn entities_for_namespace(
163 &self,
164 namespace: &str,
165 ) -> Result<Arc<dyn khive_storage::EntityStore>, SqliteError> {
166 if namespace.trim().is_empty() {
167 return Err(SqliteError::InvalidData(
168 "entities namespace must be non-empty".to_string(),
169 ));
170 }
171 let writer = self.pool.try_writer()?;
172 entity::ensure_entities_schema(writer.conn())?;
173
174 Ok(Arc::new(entity::SqlEntityStore::new(
175 Arc::clone(&self.pool),
176 self.is_file_backed,
177 )))
178 }
179
180 pub fn graph(&self) -> Result<Arc<dyn khive_storage::GraphStore>, SqliteError> {
185 self.graph_for_namespace("local")
186 }
187
188 pub fn graph_for_namespace(
190 &self,
191 namespace: &str,
192 ) -> Result<Arc<dyn khive_storage::GraphStore>, SqliteError> {
193 if namespace.trim().is_empty() {
194 return Err(SqliteError::InvalidData(
195 "graph namespace must be non-empty".to_string(),
196 ));
197 }
198 let writer = self.pool.try_writer()?;
199 graph::ensure_graph_schema(writer.conn())?;
200
201 Ok(Arc::new(graph::SqlGraphStore::new_scoped(
202 Arc::clone(&self.pool),
203 self.is_file_backed,
204 namespace.trim().to_string(),
205 )))
206 }
207
208 pub fn notes(&self) -> Result<Arc<dyn khive_storage::NoteStore>, SqliteError> {
212 self.notes_for_namespace("local")
213 }
214
215 pub fn notes_for_namespace(
219 &self,
220 namespace: &str,
221 ) -> Result<Arc<dyn khive_storage::NoteStore>, SqliteError> {
222 if namespace.trim().is_empty() {
223 return Err(SqliteError::InvalidData(
224 "notes namespace must be non-empty".to_string(),
225 ));
226 }
227 let writer = self.pool.try_writer()?;
228 note::ensure_notes_schema(writer.conn())?;
229
230 if self.notes_seq_repair_runs.load(Ordering::Relaxed) == 0 {
237 note::repair_notes_seq(writer.conn())?;
238 self.notes_seq_repair_runs.fetch_add(1, Ordering::Relaxed);
239 }
240
241 Ok(Arc::new(note::SqlNoteStore::new(
242 Arc::clone(&self.pool),
243 self.is_file_backed,
244 )))
245 }
246
247 pub fn notes_seq_repair_run_count(&self) -> usize {
252 self.notes_seq_repair_runs.load(Ordering::Relaxed)
253 }
254
255 pub fn events(&self) -> Result<Arc<dyn khive_storage::EventStore>, SqliteError> {
260 self.events_for_namespace("local")
261 }
262
263 pub fn events_for_namespace(
265 &self,
266 namespace: &str,
267 ) -> Result<Arc<dyn khive_storage::EventStore>, SqliteError> {
268 if namespace.trim().is_empty() {
269 return Err(SqliteError::InvalidData(
270 "events namespace must be non-empty".to_string(),
271 ));
272 }
273 let writer = self.pool.try_writer()?;
274 event::ensure_events_schema(writer.conn())?;
275
276 Ok(Arc::new(event::SqlEventStore::new_scoped(
277 Arc::clone(&self.pool),
278 self.is_file_backed,
279 namespace.trim().to_string(),
280 )))
281 }
282
283 pub fn vectors(
289 &self,
290 model_key: &str,
291 embedding_model: &str,
292 dimensions: usize,
293 ) -> Result<Arc<dyn khive_storage::VectorStore>, SqliteError> {
294 self.vectors_for_namespace(model_key, embedding_model, dimensions, "local")
295 }
296
297 pub fn vectors_for_namespace(
307 &self,
308 model_key: &str,
309 embedding_model: &str,
310 dimensions: usize,
311 namespace: &str,
312 ) -> Result<Arc<dyn khive_storage::VectorStore>, SqliteError> {
313 if model_key.is_empty()
314 || !model_key
315 .chars()
316 .all(|c| c.is_ascii_alphanumeric() || c == '_')
317 {
318 return Err(SqliteError::InvalidData(format!(
319 "invalid model_key '{}': must be non-empty and contain only \
320 alphanumeric/underscore characters",
321 model_key
322 )));
323 }
324 if namespace.trim().is_empty() {
325 return Err(SqliteError::InvalidData(
326 "vector store namespace must be non-empty".to_string(),
327 ));
328 }
329
330 crate::extension::ensure_extensions_loaded();
332
333 let table = format!("vec_{}", model_key);
334 let writer = self.pool.try_writer()?;
335
336 let table_exists: bool = writer
343 .conn()
344 .query_row(
345 "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1",
346 rusqlite::params![&table],
347 |row| row.get::<_, i64>(0),
348 )
349 .optional()
350 .map_err(SqliteError::Rusqlite)?
351 .is_some();
352
353 if table_exists {
354 let pragma = format!("PRAGMA table_xinfo({})", table);
360 let mut stmt = writer.conn().prepare(&pragma)?;
361 let mut rows = stmt.query([])?;
362 let mut has_field = false;
363 let mut has_embedding_model = false;
364 while let Some(row) = rows.next()? {
365 let name: String = row.get(1)?;
366 if name == "field" {
367 has_field = true;
368 }
369 if name == "embedding_model" {
370 has_embedding_model = true;
371 }
372 }
373 if !has_field || !has_embedding_model {
374 return Err(SqliteError::InvalidData(format!(
375 "vec0 table '{}' is missing required column(s) (field={}, \
376 embedding_model={}); this is a pre-v0.2.8 vector schema and is \
377 not supported — recreate the database",
378 table, has_field, has_embedding_model,
379 )));
380 }
381 }
382
383 writer
392 .conn()
393 .execute_batch(crate::migrations::EMBEDDING_MODELS_DDL)?;
394
395 writer
399 .conn()
400 .execute_batch(crate::migrations::ANN_WRITE_LOG_DDL)?;
401 writer
402 .conn()
403 .execute_batch(crate::migrations::ANN_WRITE_LOG_MODEL_SEQ_INDEX_DDL)?;
404
405 let ddl = format!(
408 "CREATE VIRTUAL TABLE IF NOT EXISTS vec_{} USING vec0(\
409 subject_id TEXT PRIMARY KEY, \
410 namespace TEXT NOT NULL, \
411 kind TEXT NOT NULL, \
412 field TEXT NOT NULL, \
413 embedding_model TEXT NOT NULL, \
414 embedding float[{}] distance_metric=cosine\
415 )",
416 model_key, dimensions
417 );
418 writer.conn().execute_batch(&ddl)?;
419
420 Ok(Arc::new(vectors::SqliteVecStore::new(
421 Arc::clone(&self.pool),
422 self.is_file_backed,
423 model_key.to_string(),
424 embedding_model.to_string(),
425 dimensions,
426 namespace.trim().to_string(),
427 )?))
428 }
429
430 pub fn register_embedding_model(
435 &self,
436 engine_name: &str,
437 model_id: &str,
438 key_version: &str,
439 dimensions: u32,
440 ) -> Result<(), SqliteError> {
441 let writer = self.pool.try_writer()?;
442 writer
443 .conn()
444 .execute_batch(crate::migrations::EMBEDDING_MODELS_DDL)?;
445
446 let now = chrono::Utc::now().timestamp_micros();
447 let canonical_key =
448 format!("{engine_name}:{model_id}:{key_version}:{dimensions}").into_bytes();
449 let id = uuid::Uuid::new_v4();
450 writer.conn().execute(
451 "INSERT INTO _embedding_models \
452 (id, engine_name, model_id, key_version, dim, output_dim, status, \
453 activated_at, superseded_at, superseded_by, canonical_key, created_at) \
454 VALUES (?1, ?2, ?3, ?4, ?5, NULL, 'active', ?6, NULL, NULL, ?7, ?8) \
455 ON CONFLICT(canonical_key) DO UPDATE SET \
456 status = 'active', \
457 activated_at = COALESCE(_embedding_models.activated_at, excluded.activated_at)",
458 rusqlite::params![
459 id.as_bytes().as_slice(),
460 engine_name,
461 model_id,
462 key_version,
463 dimensions as i64,
464 now,
465 canonical_key,
466 now,
467 ],
468 )?;
469 Ok(())
470 }
471
472 pub fn sparse(
476 &self,
477 model_key: &str,
478 ) -> Result<Arc<dyn khive_storage::SparseStore>, SqliteError> {
479 self.sparse_for_namespace(model_key, "local")
480 }
481
482 pub fn sparse_for_namespace(
486 &self,
487 model_key: &str,
488 namespace: &str,
489 ) -> Result<Arc<dyn khive_storage::SparseStore>, SqliteError> {
490 if model_key.is_empty()
491 || !model_key
492 .chars()
493 .all(|c| c.is_ascii_alphanumeric() || c == '_')
494 {
495 return Err(SqliteError::InvalidData(format!(
496 "invalid model_key '{}': must be non-empty and contain only alphanumeric/underscore characters",
497 model_key
498 )));
499 }
500 if namespace.trim().is_empty() {
501 return Err(SqliteError::InvalidData(
502 "sparse store namespace must be non-empty".to_string(),
503 ));
504 }
505
506 let writer = self.pool.try_writer()?;
507 sparse::ensure_sparse_schema(writer.conn(), model_key).map_err(SqliteError::Rusqlite)?;
508
509 Ok(Arc::new(sparse::SqliteSparseStore::new(
510 Arc::clone(&self.pool),
511 self.is_file_backed,
512 model_key.to_string(),
513 namespace.trim().to_string(),
514 )?))
515 }
516
517 pub fn text(&self, table_key: &str) -> Result<Arc<dyn khive_storage::TextSearch>, SqliteError> {
524 self.text_with_tokenizer(table_key, "trigram")
525 }
526
527 pub fn text_with_tokenizer(
535 &self,
536 table_key: &str,
537 tokenizer: &str,
538 ) -> Result<Arc<dyn khive_storage::TextSearch>, SqliteError> {
539 if table_key.is_empty()
540 || !table_key
541 .chars()
542 .all(|c| c.is_ascii_alphanumeric() || c == '_')
543 {
544 return Err(SqliteError::InvalidData(format!(
545 "invalid table_key '{}': must be non-empty and contain only \
546 alphanumeric/underscore characters",
547 table_key
548 )));
549 }
550 if tokenizer.is_empty()
551 || !tokenizer
552 .chars()
553 .all(|c| c.is_ascii_alphanumeric() || c == '_')
554 {
555 return Err(SqliteError::InvalidData(format!(
556 "invalid tokenizer '{}': must be non-empty and contain only \
557 alphanumeric/underscore characters",
558 tokenizer
559 )));
560 }
561
562 let ddl = format!(
563 "CREATE VIRTUAL TABLE IF NOT EXISTS fts_{} USING fts5(\
564 subject_id UNINDEXED, \
565 kind UNINDEXED, \
566 title, \
567 body, \
568 tags UNINDEXED, \
569 namespace UNINDEXED, \
570 metadata UNINDEXED, \
571 updated_at UNINDEXED, \
572 tokenize = '{}'\
573 )",
574 table_key, tokenizer
575 );
576 let writer = self.pool.try_writer()?;
577 writer.conn().execute_batch(&ddl)?;
578
579 Ok(Arc::new(text::Fts5TextSearch::new(
580 Arc::clone(&self.pool),
581 self.is_file_backed,
582 table_key.to_string(),
583 )))
584 }
585
586 pub fn blob_store(
594 &self,
595 config_root: Option<&Path>,
596 floor_bytes: Option<u64>,
597 ) -> Result<Arc<dyn khive_storage::BlobStore>, SqliteError> {
598 let root = blob::resolve_blob_root(self.data_dir().as_deref(), config_root)?;
599 let floor = floor_bytes.unwrap_or(blob::FsBlobStore::DEFAULT_FLOOR_BYTES);
600 Ok(Arc::new(blob::FsBlobStore::new(root, floor)?))
601 }
602
603 pub fn is_file_backed(&self) -> bool {
605 self.is_file_backed
606 }
607
608 pub fn data_dir(&self) -> Option<std::path::PathBuf> {
611 self.path.as_ref()?.parent().map(|p| p.to_path_buf())
612 }
613
614 pub fn ann_root(&self) -> Option<std::path::PathBuf> {
622 ann_root_for(self.path.as_ref()?)
623 }
624
625 pub fn pool(&self) -> &ConnectionPool {
627 &self.pool
628 }
629
630 pub fn pool_arc(&self) -> Arc<ConnectionPool> {
632 Arc::clone(&self.pool)
633 }
634}
635
636fn ann_root_for(path: &std::path::Path) -> Option<std::path::PathBuf> {
641 let mut file = path.file_name()?.to_os_string();
642 file.push(".ann");
643 path.parent().map(|p| p.join(file))
644}
645
646#[cfg(test)]
647mod tests {
648 use super::*;
649 use khive_storage::types::{SqlStatement, SqlValue};
650
651 #[test]
652 fn memory_backend_creates_successfully() {
653 let backend = StorageBackend::memory().expect("memory backend should create");
654 assert!(!backend.is_file_backed());
655 }
656
657 #[test]
658 fn file_backend_creates_successfully() {
659 let dir = tempfile::tempdir().unwrap();
660 let path = dir.path().join("test.db");
661 let backend = StorageBackend::sqlite(&path).expect("file backend should create");
662 assert!(backend.is_file_backed());
663 assert!(path.exists());
664 }
665
666 #[test]
667 fn data_dir_returns_none_for_memory_backend() {
668 let backend = StorageBackend::memory().expect("memory backend");
669 assert!(backend.data_dir().is_none());
670 }
671
672 #[test]
673 fn data_dir_returns_parent_dir_for_file_backend() {
674 let dir = tempfile::tempdir().unwrap();
675 let path = dir.path().join("data.db");
676 let backend = StorageBackend::sqlite(&path).expect("file backend");
677 let got = backend.data_dir().expect("file backend must return Some");
678 assert_eq!(got, dir.path());
679 }
680
681 #[test]
682 fn ann_root_is_database_scoped_sibling_dir() {
683 let dir = tempfile::tempdir().unwrap();
684 let path = dir.path().join("data.db");
685 let backend = StorageBackend::sqlite(&path).expect("file backend");
686 let got = backend.ann_root().expect("file backend must return Some");
687 assert_eq!(got, dir.path().join("data.db.ann"));
688 assert!(StorageBackend::memory().unwrap().ann_root().is_none());
689 }
690
691 #[cfg(unix)]
697 #[test]
698 fn ann_root_distinct_for_non_utf8_filenames() {
699 use std::os::unix::ffi::OsStrExt;
700 let path_a = std::path::Path::new("/data").join(std::ffi::OsStr::from_bytes(b"\xff.db"));
701 let path_b = std::path::Path::new("/data").join(std::ffi::OsStr::from_bytes(b"\xfe.db"));
702 let root_a = ann_root_for(&path_a).expect("Some for a file path");
703 let root_b = ann_root_for(&path_b).expect("Some for a file path");
704 assert_ne!(
705 root_a, root_b,
706 "distinct database files must map to distinct ANN roots"
707 );
708 }
709
710 #[tokio::test]
711 async fn sql_access_memory_roundtrip() {
712 let backend = StorageBackend::memory().unwrap();
713 let sql = backend.sql();
714
715 let mut writer = sql.writer().await.unwrap();
716 writer
717 .execute_script(
718 "CREATE TABLE test_rt (id TEXT PRIMARY KEY, value INTEGER NOT NULL)".into(),
719 )
720 .await
721 .unwrap();
722
723 let affected = writer
724 .execute(SqlStatement {
725 sql: "INSERT INTO test_rt (id, value) VALUES (?1, ?2)".into(),
726 params: vec![SqlValue::Text("row1".into()), SqlValue::Integer(42)],
727 label: None,
728 })
729 .await
730 .unwrap();
731 assert_eq!(affected, 1);
732
733 let mut reader = sql.reader().await.unwrap();
734 let row = reader
735 .query_row(SqlStatement {
736 sql: "SELECT id, value FROM test_rt WHERE id = ?1".into(),
737 params: vec![SqlValue::Text("row1".into())],
738 label: None,
739 })
740 .await
741 .unwrap();
742
743 let row = row.expect("should find the inserted row");
744 assert_eq!(row.columns.len(), 2);
745 match &row.columns[0].value {
746 SqlValue::Text(s) => assert_eq!(s, "row1"),
747 other => panic!("expected Text, got {other:?}"),
748 }
749 match &row.columns[1].value {
750 SqlValue::Integer(v) => assert_eq!(*v, 42),
751 other => panic!("expected Integer, got {other:?}"),
752 }
753 }
754
755 #[tokio::test]
756 async fn sql_access_file_roundtrip() {
757 let dir = tempfile::tempdir().unwrap();
758 let path = dir.path().join("test_roundtrip.db");
759 let backend = StorageBackend::sqlite(&path).unwrap();
760 let sql = backend.sql();
761
762 let mut writer = sql.writer().await.unwrap();
763 writer
764 .execute_script("CREATE TABLE test_f (k TEXT PRIMARY KEY, v TEXT)".into())
765 .await
766 .unwrap();
767 writer
768 .execute(SqlStatement {
769 sql: "INSERT INTO test_f (k, v) VALUES (?1, ?2)".into(),
770 params: vec![
771 SqlValue::Text("hello".into()),
772 SqlValue::Text("world".into()),
773 ],
774 label: None,
775 })
776 .await
777 .unwrap();
778
779 let mut reader = sql.reader().await.unwrap();
780 let rows = reader
781 .query_all(SqlStatement {
782 sql: "SELECT k, v FROM test_f".into(),
783 params: vec![],
784 label: None,
785 })
786 .await
787 .unwrap();
788 assert_eq!(rows.len(), 1);
789 match &rows[0].columns[1].value {
790 SqlValue::Text(s) => assert_eq!(s, "world"),
791 other => panic!("expected Text, got {other:?}"),
792 }
793 }
794
795 #[test]
796 fn sqlite_read_only_missing_path_does_not_create_file() {
797 let dir = tempfile::tempdir().unwrap();
798 let path = dir.path().join("missing_ro.db");
799 assert!(!path.exists());
800
801 let result = StorageBackend::sqlite_read_only(&path);
802 assert!(
803 result.is_err(),
804 "opening a missing path read-only must fail"
805 );
806 assert!(
807 !path.exists(),
808 "opening a missing path read-only must not create the file"
809 );
810 }
811
812 #[tokio::test]
813 async fn sqlite_read_only_sql_writer_rejects_ddl_and_insert() {
814 let dir = tempfile::tempdir().unwrap();
815 let path = dir.path().join("ro_writer.db");
816
817 {
819 let writable = StorageBackend::sqlite(&path).unwrap();
820 let sql = writable.sql();
821 let mut writer = sql.writer().await.unwrap();
822 writer
823 .execute_script("CREATE TABLE ro_existing (id INTEGER PRIMARY KEY)".into())
824 .await
825 .unwrap();
826 }
827
828 let ro = StorageBackend::sqlite_read_only(&path).unwrap();
829 let sql = ro.sql();
830
831 let writer_result = sql.writer().await;
833 assert!(
834 writer_result.is_err(),
835 "sql().writer() must be rejected on a read-only backend"
836 );
837 }
838
839 #[tokio::test]
840 #[cfg(feature = "vectors")]
841 async fn vectors_roundtrip_via_public_api() {
842 let backend = StorageBackend::memory().unwrap();
843 let store = backend.vectors("test_api", "test_api", 3).unwrap();
844
845 let id = uuid::Uuid::new_v4();
846 store
847 .insert(
848 id,
849 khive_types::SubstrateKind::Entity,
850 "local",
851 "content",
852 vec![vec![1.0, 0.0, 0.0]],
853 )
854 .await
855 .unwrap();
856
857 let hits = store
858 .search(khive_storage::types::VectorSearchRequest {
859 query_vectors: vec![vec![1.0, 0.0, 0.0]],
860 top_k: 1,
861 namespace: None,
862 kind: None,
863 embedding_model: None,
864 filter: None,
865 backend_hints: None,
866 })
867 .await
868 .unwrap();
869
870 assert_eq!(hits.len(), 1);
871 assert_eq!(hits[0].subject_id, id);
872 assert!(hits[0].score.to_f64() > 0.99);
873 }
874
875 #[tokio::test]
876 #[cfg(feature = "vectors")]
877 async fn vectors_creates_table_idempotently() {
878 let backend = StorageBackend::memory().unwrap();
879
880 let store1 = backend.vectors("idempotent", "idempotent", 3).unwrap();
881 let store2 = backend.vectors("idempotent", "idempotent", 3).unwrap();
882
883 let id = uuid::Uuid::new_v4();
884 store1
885 .insert(
886 id,
887 khive_types::SubstrateKind::Entity,
888 "local",
889 "content",
890 vec![vec![1.0, 0.0, 0.0]],
891 )
892 .await
893 .unwrap();
894
895 let count = store2.count().await.unwrap();
896 assert_eq!(count, 1);
897 }
898
899 #[tokio::test]
900 async fn text_roundtrip_via_public_api() {
901 let backend = StorageBackend::memory().unwrap();
902 let store = backend.text("test_api").unwrap();
903
904 let id = uuid::Uuid::new_v4();
905 let doc = khive_storage::types::TextDocument {
906 subject_id: id,
907 kind: khive_types::SubstrateKind::Entity,
908 title: Some("Test Title".to_string()),
909 body: "This is a searchable document about Rust.".to_string(),
910 tags: vec!["rust".to_string()],
911 namespace: "test_ns".to_string(),
912 metadata: None,
913 updated_at: chrono::Utc::now(),
914 };
915 store.upsert_document(doc).await.unwrap();
916
917 let hits = store
918 .search(khive_storage::types::TextSearchRequest {
919 query: "Rust".to_string(),
920 mode: khive_storage::types::TextQueryMode::Plain,
921 filter: Some(khive_storage::types::TextFilter {
922 namespaces: vec!["test_ns".to_string()],
923 ..Default::default()
924 }),
925 top_k: 1,
926 snippet_chars: 64,
927 })
928 .await
929 .unwrap();
930
931 assert_eq!(hits.len(), 1);
932 assert_eq!(hits[0].subject_id, id);
933 assert!(hits[0].score.to_f64() > 0.0);
934 }
935
936 #[tokio::test]
937 async fn text_creates_table_idempotently() {
938 let backend = StorageBackend::memory().unwrap();
939
940 let store1 = backend.text("idempotent_fts").unwrap();
941 let store2 = backend.text("idempotent_fts").unwrap();
942
943 let id = uuid::Uuid::new_v4();
944 let doc = khive_storage::types::TextDocument {
945 subject_id: id,
946 kind: khive_types::SubstrateKind::Note,
947 title: None,
948 body: "Hello world.".to_string(),
949 tags: vec![],
950 namespace: "test_ns".to_string(),
951 metadata: None,
952 updated_at: chrono::Utc::now(),
953 };
954 store1.upsert_document(doc).await.unwrap();
955
956 let count = store2
957 .count(khive_storage::types::TextFilter {
958 namespaces: vec!["test_ns".to_string()],
959 ..Default::default()
960 })
961 .await
962 .unwrap();
963 assert_eq!(count, 1);
964 }
965
966 #[test]
967 fn invalid_model_key_rejected() {
968 let backend = StorageBackend::memory().unwrap();
969 assert!(backend.vectors("bad key!", "bad key!", 3).is_err());
970 assert!(backend.vectors("", "", 3).is_err());
971 }
972
973 #[test]
974 fn invalid_table_key_rejected() {
975 let backend = StorageBackend::memory().unwrap();
976 assert!(backend.text("bad key!").is_err());
977 assert!(backend.text("").is_err());
978 }
979
980 #[tokio::test]
981 async fn sqlite_read_only_graph_store_rejects_upsert_edge() {
982 use khive_storage::types::Edge;
983 use khive_types::EdgeRelation;
984
985 let dir = tempfile::tempdir().unwrap();
986 let path = dir.path().join("ro_graph.db");
987
988 {
990 let writable = StorageBackend::sqlite(&path).unwrap();
991 writable.graph().unwrap();
992 }
993
994 let ro = StorageBackend::sqlite_read_only(&path).unwrap();
995 let store = match ro.graph() {
996 Ok(store) => store,
997 Err(_) => return,
1000 };
1001
1002 let now = chrono::Utc::now();
1003 let edge = Edge {
1004 id: uuid::Uuid::new_v4().into(),
1005 namespace: "local".to_string(),
1006 source_id: uuid::Uuid::new_v4(),
1007 target_id: uuid::Uuid::new_v4(),
1008 relation: EdgeRelation::Extends,
1009 weight: 0.8,
1010 created_at: now,
1011 updated_at: now,
1012 deleted_at: None,
1013 metadata: None,
1014 target_backend: None,
1015 };
1016
1017 let result = store.upsert_edge(edge).await;
1018 assert!(
1019 result.is_err(),
1020 "upsert_edge on a read-only backend must reject, not silently no-op"
1021 );
1022 }
1023
1024 #[tokio::test]
1025 async fn sqlite_read_only_event_store_rejects_append_event() {
1026 use khive_types::{EventKind, EventOutcome, SubstrateKind};
1027
1028 let dir = tempfile::tempdir().unwrap();
1029 let path = dir.path().join("ro_events.db");
1030
1031 {
1032 let writable = StorageBackend::sqlite(&path).unwrap();
1033 writable.events().unwrap();
1034 }
1035
1036 let ro = StorageBackend::sqlite_read_only(&path).unwrap();
1037 let store = match ro.events() {
1038 Ok(store) => store,
1039 Err(_) => return,
1040 };
1041
1042 let event = khive_storage::event::Event::new(
1043 "local",
1044 "test.verb",
1045 EventKind::Audit,
1046 SubstrateKind::Entity,
1047 "test-actor",
1048 )
1049 .with_outcome(EventOutcome::Success);
1050
1051 let result = store.append_event(event).await;
1052 assert!(
1053 result.is_err(),
1054 "append_event on a read-only backend must reject, not silently no-op"
1055 );
1056 }
1057
1058 #[tokio::test]
1059 async fn sqlite_read_only_text_store_rejects_upsert_document() {
1060 use khive_storage::types::TextDocument;
1061 use khive_types::SubstrateKind;
1062
1063 let dir = tempfile::tempdir().unwrap();
1064 let path = dir.path().join("ro_text.db");
1065
1066 {
1067 let writable = StorageBackend::sqlite(&path).unwrap();
1068 writable.text("ro_test").unwrap();
1069 }
1070
1071 let ro = StorageBackend::sqlite_read_only(&path).unwrap();
1072 let store = match ro.text("ro_test") {
1073 Ok(store) => store,
1074 Err(_) => return,
1075 };
1076
1077 let doc = TextDocument {
1078 subject_id: uuid::Uuid::new_v4(),
1079 kind: SubstrateKind::Entity,
1080 title: Some("Title".to_string()),
1081 body: "Body text.".to_string(),
1082 tags: vec![],
1083 namespace: "local".to_string(),
1084 metadata: None,
1085 updated_at: chrono::Utc::now(),
1086 };
1087
1088 let result = store.upsert_document(doc).await;
1089 assert!(
1090 result.is_err(),
1091 "upsert_document on a read-only backend must reject, not silently no-op"
1092 );
1093 }
1094
1095 #[tokio::test]
1096 async fn blob_store_roundtrip_via_public_api() {
1097 let dir = tempfile::tempdir().unwrap();
1098 let path = dir.path().join("blob_backend.db");
1099 let backend = StorageBackend::sqlite(&path).unwrap();
1100
1101 let store = backend.blob_store(None, Some(0)).unwrap();
1105 let bytes = b"backend-level blob roundtrip".to_vec();
1106 let content_ref = store.put(bytes.clone()).await.unwrap();
1107 assert_eq!(store.get(&content_ref).await.unwrap(), bytes);
1108 }
1109
1110 #[test]
1111 fn blob_store_defaults_root_beside_db_file() {
1112 let dir = tempfile::tempdir().unwrap();
1113 let path = dir.path().join("blob_default.db");
1114 let backend = StorageBackend::sqlite(&path).unwrap();
1115
1116 let _store = backend.blob_store(None, None).unwrap();
1120 assert!(
1121 dir.path().join("blobs").is_dir(),
1122 "default root must be created beside the database file"
1123 );
1124 }
1125
1126 #[test]
1127 fn blob_store_errors_for_in_memory_backend_with_no_override() {
1128 let backend = StorageBackend::memory().unwrap();
1129 assert!(backend.blob_store(None, None).is_err());
1130 }
1131
1132 #[test]
1133 fn blob_store_accepts_explicit_root_for_in_memory_backend() {
1134 let dir = tempfile::tempdir().unwrap();
1135 let backend = StorageBackend::memory().unwrap();
1136 let store = backend.blob_store(Some(dir.path()), None);
1137 assert!(store.is_ok());
1138 }
1139
1140 #[test]
1141 fn apply_schema_runs_migrations_idempotently() {
1142 static MIGRATIONS: &[crate::migrations::Migration] = &[crate::migrations::Migration {
1143 id: "001_init",
1144 up_sql: "CREATE TABLE IF NOT EXISTS schema_test (id TEXT PRIMARY KEY);",
1145 down_sql: None,
1146 is_already_applied: None,
1147 }];
1148 let plan = crate::migrations::ServiceSchemaPlan {
1149 service: "schema_test_svc",
1150 sqlite: MIGRATIONS,
1151 postgres: &[],
1152 };
1153
1154 let backend = StorageBackend::memory().unwrap();
1155 backend.apply_schema(&plan).unwrap();
1156 backend.apply_schema(&plan).unwrap();
1157
1158 let reader = backend.pool().reader().unwrap();
1159 let count: i64 = reader
1160 .conn()
1161 .query_row(
1162 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_test'",
1163 [],
1164 |row| row.get(0),
1165 )
1166 .unwrap();
1167 assert_eq!(count, 1);
1168 }
1169
1170 fn issue_1029_pool(write_queue_enabled: bool) -> (tempfile::TempDir, StorageBackend) {
1179 let dir = tempfile::tempdir().unwrap();
1180 let path = dir.path().join("issue_1029.db");
1181 let config = crate::pool::PoolConfig {
1182 path: Some(path.clone()),
1183 busy_timeout: std::time::Duration::from_millis(200),
1184 write_queue_enabled,
1185 ..crate::pool::PoolConfig::default()
1186 };
1187 let pool = ConnectionPool::new(config).expect("fresh tenant-shaped pool should open");
1188 let backend = StorageBackend {
1189 pool: Arc::new(pool),
1190 is_file_backed: true,
1191 path: Some(path),
1192 notes_seq_repair_runs: AtomicUsize::new(0),
1193 };
1194 (dir, backend)
1195 }
1196
1197 async fn issue_1029_create_entity_shaped_sequence(
1198 backend: &StorageBackend,
1199 ) -> Result<(), String> {
1200 let entities = backend
1201 .entities_for_namespace("tenant_ns")
1202 .map_err(|e| format!("entities_for_namespace: {e}"))?;
1203 let entity = khive_storage::entity::Entity::new("tenant_ns", "concept", "Issue1029Repro");
1204 let entity_id = entity.id;
1205 entities
1206 .upsert_entity(entity)
1207 .await
1208 .map_err(|e| format!("upsert_entity: {e}"))?;
1209
1210 let text = backend.text("entities").map_err(|e| format!("text: {e}"))?;
1211 let doc = khive_storage::types::TextDocument {
1212 subject_id: entity_id,
1213 kind: khive_types::SubstrateKind::Entity,
1214 title: Some("Issue1029Repro".to_string()),
1215 body: "issue 1029 repro body".to_string(),
1216 tags: vec![],
1217 namespace: "tenant_ns".to_string(),
1218 metadata: None,
1219 updated_at: chrono::Utc::now(),
1220 };
1221 text.upsert_document(doc)
1222 .await
1223 .map_err(|e| format!("fts_upsert: {e}"))
1224 }
1225
1226 #[tokio::test]
1232 async fn issue_1029_create_entity_shaped_sequence_write_queue_off() {
1233 let (_dir, backend) = issue_1029_pool(false);
1234 let result = issue_1029_create_entity_shaped_sequence(&backend).await;
1235 assert!(
1236 result.is_ok(),
1237 "khive#1029 repro (KHIVE_WRITE_QUEUE off): fts_upsert step failed: {:?}",
1238 result.err()
1239 );
1240 }
1241
1242 #[tokio::test]
1248 async fn issue_1029_create_entity_shaped_sequence_write_queue_on() {
1249 let (_dir, backend) = issue_1029_pool(true);
1250 let result = issue_1029_create_entity_shaped_sequence(&backend).await;
1251 assert!(
1252 result.is_ok(),
1253 "khive#1029 repro (KHIVE_WRITE_QUEUE=1): fts_upsert step failed: {:?}",
1254 result.err()
1255 );
1256 }
1257
1258 #[tokio::test]
1266 async fn issue_1029_two_pools_same_file_write_queue_on() {
1267 let dir = tempfile::tempdir().unwrap();
1268 let path = dir.path().join("issue_1029_two_pools.db");
1269
1270 let cfg = |p: std::path::PathBuf| crate::pool::PoolConfig {
1271 path: Some(p),
1272 busy_timeout: std::time::Duration::from_millis(200),
1273 write_queue_enabled: true,
1274 ..crate::pool::PoolConfig::default()
1275 };
1276
1277 let pool_a = ConnectionPool::new(cfg(path.clone())).expect("pool A should open");
1278 let backend_a = StorageBackend {
1279 pool: Arc::new(pool_a),
1280 is_file_backed: true,
1281 path: Some(path.clone()),
1282 notes_seq_repair_runs: AtomicUsize::new(0),
1283 };
1284 let pool_b = ConnectionPool::new(cfg(path.clone())).expect("pool B should open");
1285 let backend_b = StorageBackend {
1286 pool: Arc::new(pool_b),
1287 is_file_backed: true,
1288 path: Some(path),
1289 notes_seq_repair_runs: AtomicUsize::new(0),
1290 };
1291
1292 let entities = backend_a
1293 .entities_for_namespace("tenant_ns")
1294 .expect("entities_for_namespace on pool A");
1295 let entity =
1296 khive_storage::entity::Entity::new("tenant_ns", "concept", "Issue1029TwoPools");
1297 let entity_id = entity.id;
1298 entities
1299 .upsert_entity(entity)
1300 .await
1301 .expect("pool A entity upsert should succeed");
1302
1303 let text = backend_b.text("entities").expect("text on pool B");
1304 let doc = khive_storage::types::TextDocument {
1305 subject_id: entity_id,
1306 kind: khive_types::SubstrateKind::Entity,
1307 title: Some("Issue1029TwoPools".to_string()),
1308 body: "issue 1029 two-pool repro body".to_string(),
1309 tags: vec![],
1310 namespace: "tenant_ns".to_string(),
1311 metadata: None,
1312 updated_at: chrono::Utc::now(),
1313 };
1314 let result = text.upsert_document(doc).await;
1315 assert!(
1316 result.is_ok(),
1317 "khive#1029 two-pool repro: fts_upsert on an independent pool for the \
1318 same tenant DB file failed: {:?}",
1319 result.err()
1320 );
1321 }
1322
1323 struct StarvationCaptureSubscriber {
1326 events: Arc<std::sync::Mutex<Vec<std::collections::BTreeMap<String, String>>>>,
1327 }
1328
1329 impl tracing::Subscriber for StarvationCaptureSubscriber {
1330 fn enabled(&self, _: &tracing::Metadata<'_>) -> bool {
1331 true
1332 }
1333 fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id {
1334 tracing::span::Id::from_u64(1)
1335 }
1336 fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
1337 fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
1338 fn event(&self, event: &tracing::Event<'_>) {
1339 #[derive(Default)]
1340 struct FieldVisitor(std::collections::BTreeMap<String, String>);
1341 impl tracing::field::Visit for FieldVisitor {
1342 fn record_debug(
1343 &mut self,
1344 field: &tracing::field::Field,
1345 value: &dyn std::fmt::Debug,
1346 ) {
1347 self.0
1348 .insert(field.name().to_string(), format!("{value:?}"));
1349 }
1350 }
1351 let mut visitor = FieldVisitor::default();
1352 event.record(&mut visitor);
1353 self.events.lock().unwrap().push(visitor.0);
1354 }
1355 fn enter(&self, _: &tracing::span::Id) {}
1356 fn exit(&self, _: &tracing::span::Id) {}
1357 }
1358
1359 #[tokio::test]
1372 #[serial_test::serial(tx_registry)]
1373 async fn issue_1029_starvation_warn_reports_registered_transactions() {
1374 let (_dir, backend) = issue_1029_pool(false);
1375 let text = backend.text("entities").expect("text store");
1378
1379 let holder = backend
1383 .pool
1384 .open_standalone_writer()
1385 .expect("holder connection");
1386 holder
1387 .execute_batch("BEGIN IMMEDIATE")
1388 .expect("holder BEGIN IMMEDIATE");
1389 let fixture =
1390 khive_storage::tx_registry::register(Some("issue_1029_fixture_tx".to_string()));
1391
1392 let events = Arc::new(std::sync::Mutex::new(Vec::new()));
1393 let subscriber = StarvationCaptureSubscriber {
1394 events: Arc::clone(&events),
1395 };
1396 let guard = tracing::subscriber::set_default(subscriber);
1397
1398 let doc = khive_storage::types::TextDocument {
1399 subject_id: uuid::Uuid::new_v4(),
1400 kind: khive_types::SubstrateKind::Entity,
1401 title: Some("Issue1029Starved".to_string()),
1402 body: "issue 1029 starvation diagnostic body".to_string(),
1403 tags: vec![],
1404 namespace: "tenant_ns".to_string(),
1405 metadata: None,
1406 updated_at: chrono::Utc::now(),
1407 };
1408 let result = text.upsert_document(doc).await;
1409
1410 drop(guard);
1411 drop(fixture);
1412 holder
1413 .execute_batch("ROLLBACK")
1414 .expect("holder ROLLBACK releases the lock");
1415
1416 assert!(
1417 result.is_err(),
1418 "upsert_document must starve while another connection holds the write lock"
1419 );
1420
1421 let events = events.lock().unwrap();
1422 let warn = events
1423 .iter()
1424 .find(|fields| {
1425 fields
1426 .get("message")
1427 .is_some_and(|m| m.contains("text write starved"))
1428 })
1429 .unwrap_or_else(|| panic!("expected a starvation WARN, captured events: {events:?}"));
1430 assert!(
1431 warn.get("op").is_some_and(|op| op.contains("fts_upsert")),
1432 "WARN must name the starved operation, got: {warn:?}"
1433 );
1434 assert!(
1435 warn.get("open_txs")
1436 .is_some_and(|txs| txs.contains("issue_1029_fixture_tx")),
1437 "WARN must list the registered holder label, got: {warn:?}"
1438 );
1439 let count: usize = warn
1440 .get("open_tx_count")
1441 .expect("WARN must carry open_tx_count")
1442 .parse()
1443 .expect("open_tx_count must be numeric");
1444 assert!(
1445 count >= 1,
1446 "open_tx_count must count the fixture, got {count}"
1447 );
1448 }
1449}