1use std::sync::Arc;
4
5use anyhow::Context as _;
6use khive_db::migrations::AttachmentCutoverStatus;
7use khive_storage::types::SqlStatement;
8use khive_storage::{Attachment, AttachmentSubstrate, SqlAccess};
9
10use crate::{BlobHydrator, LegacyPreferenceVerifier, RuntimeError, StorageBackend};
11
12const FANN_NETWORK_ROLE: &str = "fann-network";
13
14pub async fn legacy_preference_model_count(sql: &dyn SqlAccess) -> Result<u64, RuntimeError> {
24 let mut reader = sql.reader().await?;
25 Ok(reader
26 .count(SqlStatement {
27 sql: "SELECT COUNT(*) FROM entities \
28 WHERE kind = 'artifact' AND entity_type = 'moodboard_model' \
29 AND content_ref IS NOT NULL"
30 .to_string(),
31 params: vec![],
32 label: Some("moodboard_legacy_preference_model_count".to_string()),
33 })
34 .await?)
35}
36
37enum CutoverMode {
38 Main,
39 EmptySecondary { backend_name: String },
40}
41
42impl CutoverMode {
43 fn secondary_name(&self) -> Option<&str> {
44 match self {
45 Self::Main => None,
46 Self::EmptySecondary { backend_name } => Some(backend_name),
47 }
48 }
49}
50
51#[cfg(test)]
52mod test_sync {
53 use std::path::PathBuf;
54 use std::sync::{Arc, Mutex, OnceLock};
55
56 use khive_storage::SqlAccess;
57 use tokio::sync::Barrier;
58
59 struct Hook {
60 database_path: PathBuf,
61 barrier: Arc<Barrier>,
62 }
63
64 fn hook() -> &'static Mutex<Option<Hook>> {
65 static HOOK: OnceLock<Mutex<Option<Hook>>> = OnceLock::new();
66 HOOK.get_or_init(|| Mutex::new(None))
67 }
68
69 pub(super) fn install(database_path: PathBuf, barrier: Arc<Barrier>) {
70 *hook().lock().expect("cutover test hook lock") = Some(Hook {
71 database_path,
72 barrier,
73 });
74 }
75
76 pub(super) fn clear() {
77 *hook().lock().expect("cutover test hook lock") = None;
78 }
79
80 pub(super) async fn before_gc_owner(sql: &dyn SqlAccess) {
81 let barrier = hook()
82 .lock()
83 .expect("cutover test hook lock")
84 .as_ref()
85 .filter(|hook| sql.database_path().as_ref() == Some(&hook.database_path))
86 .map(|hook| Arc::clone(&hook.barrier));
87 if let Some(barrier) = barrier {
88 barrier.wait().await;
89 }
90 }
91}
92
93async fn scalar_count(
94 backend: &StorageBackend,
95 sql: &'static str,
96 label: &'static str,
97) -> anyhow::Result<u64> {
98 let sql_access = backend.sql();
99 let mut reader = sql_access
100 .reader()
101 .await
102 .with_context(|| format!("{label}: acquire SQL reader"))?;
103 reader
104 .count(SqlStatement {
105 sql: sql.to_string(),
106 params: vec![],
107 label: Some(label.to_string()),
108 })
109 .await
110 .with_context(|| format!("{label}: execute count"))
111}
112
113async fn cutover_status(backend: Arc<StorageBackend>) -> anyhow::Result<AttachmentCutoverStatus> {
114 tokio::task::spawn_blocking(move || backend.attachment_cutover_status())
115 .await
116 .context("attachment cutover status task panicked")?
117 .context("inspect attachment cutover status")
118}
119
120pub async fn require_secondary_attachment_empty(
127 backend: Arc<StorageBackend>,
128 backend_name: &str,
129) -> anyhow::Result<()> {
130 let status = cutover_status(Arc::clone(&backend)).await?;
131 let legacy_refs = match status {
132 AttachmentCutoverStatus::Pending | AttachmentCutoverStatus::Incomplete => {
133 scalar_count(
134 backend.as_ref(),
135 "SELECT COUNT(*) FROM entities WHERE content_ref IS NOT NULL",
136 "secondary_legacy_attachment_count",
137 )
138 .await?
139 }
140 AttachmentCutoverStatus::Complete => 0,
141 };
142 let attachment_rows = match status {
143 AttachmentCutoverStatus::Pending => 0,
144 AttachmentCutoverStatus::Incomplete | AttachmentCutoverStatus::Complete => {
145 scalar_count(
146 backend.as_ref(),
147 "SELECT COUNT(*) FROM attachments",
148 "secondary_attachment_row_count",
149 )
150 .await?
151 }
152 };
153
154 if legacy_refs != 0 || attachment_rows != 0 {
155 anyhow::bail!(
156 "backend {backend_name:?} contains attachment-bearing records \
157 (legacy_refs={legacy_refs}, attachment_rows={attachment_rows}); V21 makes the \
158 main backend the sole attachment/GC liveness authority. Move or remove those \
159 recoverable records before retrying boot"
160 );
161 }
162 Ok(())
163}
164
165pub async fn coordinate_attachment_cutover(
219 backend: Arc<StorageBackend>,
220 hydrator: Option<Arc<BlobHydrator>>,
221 verifier: Option<Arc<dyn LegacyPreferenceVerifier>>,
222) -> anyhow::Result<()> {
223 coordinate_attachment_cutover_inner(backend, hydrator, verifier, CutoverMode::Main).await
224}
225
226pub async fn coordinate_empty_secondary_attachment_cutover(
229 backend: Arc<StorageBackend>,
230 backend_name: &str,
231) -> anyhow::Result<()> {
232 coordinate_attachment_cutover_inner(
233 backend,
234 None,
235 None,
236 CutoverMode::EmptySecondary {
237 backend_name: backend_name.to_string(),
238 },
239 )
240 .await
241}
242
243async fn coordinate_attachment_cutover_inner(
244 backend: Arc<StorageBackend>,
245 hydrator: Option<Arc<BlobHydrator>>,
246 verifier: Option<Arc<dyn LegacyPreferenceVerifier>>,
247 mode: CutoverMode,
248) -> anyhow::Result<()> {
249 if cutover_status(Arc::clone(&backend)).await? == AttachmentCutoverStatus::Complete {
250 if let Some(backend_name) = mode.secondary_name() {
251 require_secondary_attachment_empty(backend, backend_name).await?;
256 }
257 return Ok(());
258 }
259
260 let prepared = Arc::clone(&backend);
261 let migration = khive_db::WriteClassContext::migration();
265 let handle = crate::spawn_named_tracked_task(
266 "attachment_cutover",
267 migration.scope_future(async move {
268 let sql = backend.sql();
269 #[cfg(test)]
270 test_sync::before_gc_owner(sql.as_ref()).await;
271 let owner = khive_db::stores::blob::acquire_database_gc_owner(sql.as_ref())
272 .await
273 .context("acquire canonical database GC owner for V21")?;
274
275 let (backend, owner, already_complete) = tokio::task::spawn_blocking(move || {
276 if backend.attachment_cutover_status()? == AttachmentCutoverStatus::Complete {
281 return Ok::<_, anyhow::Error>((backend, owner, true));
282 }
283 backend
284 .stage_attachment_cutover(&owner)
285 .context("commit V21 attachment stage 1")?;
286 Ok::<_, anyhow::Error>((backend, owner, false))
287 })
288 .await
289 .context("V21 stage-1 blocking task panicked")??;
290 if already_complete {
291 if let Some(backend_name) = mode.secondary_name() {
292 require_secondary_attachment_empty(backend, backend_name).await?;
293 }
294 return Ok(());
295 }
296
297 let verified = if let Some(backend_name) = mode.secondary_name() {
298 require_secondary_attachment_empty(Arc::clone(&backend), backend_name).await?;
303 Vec::new()
304 } else {
305 let sql = backend.sql();
306 let legacy_model_count = legacy_preference_model_count(sql.as_ref())
307 .await
308 .context("count legacy moodboard preference models")?;
309 if legacy_model_count == 0 {
310 Vec::new()
311 } else {
312 let verifier = verifier.as_ref().ok_or_else(|| {
313 anyhow::anyhow!(
314 "V21 found {legacy_model_count} legacy moodboard preference model(s), but \
315 this build has no moodboard support (compiled without the \
316 `pack-moodboard` feature); rebuild with that feature enabled to \
317 migrate them, then retry boot"
318 )
319 })?;
320 let hydrator = hydrator.as_ref().ok_or_else(|| {
321 anyhow::anyhow!(
322 "V21 found {legacy_model_count} legacy moodboard preference model(s), but \
323 no BlobHydrator is configured; configure [storage.blob] and retry boot"
324 )
325 })?;
326 let verified = verifier
327 .verify_legacy_preference_attachments(sql.as_ref(), hydrator.as_ref())
328 .await
329 .context("verify legacy moodboard bundle/event/FANN evidence")?;
330 if verified.len() as u64 != legacy_model_count {
331 anyhow::bail!(
332 "legacy moodboard verification returned {} attachment(s) for \
333 {legacy_model_count} recoverable model(s)",
334 verified.len()
335 );
336 }
337 verified
338 }
339 };
340
341 let created_at = chrono::Utc::now().timestamp_micros();
342 let attachments = verified
343 .into_iter()
344 .map(|verified| Attachment {
345 record_uuid: verified.model_id,
346 substrate: AttachmentSubstrate::Entity,
347 role: FANN_NETWORK_ROLE.to_string(),
348 content_ref: verified.network_content_ref,
349 media_type: Some("application/octet-stream".to_string()),
350 size_bytes: Some(verified.size_bytes),
351 created_at,
352 })
353 .collect::<Vec<_>>();
354
355 tokio::task::spawn_blocking(move || {
356 backend
357 .apply_verified_attachments(&owner, &attachments)
358 .context("atomically apply verified pack-owned attachments")?;
359 backend
360 .finalize_attachment_cutover(&owner)
361 .context("finalize V21 attachment/GC cutover")?;
362 Ok::<_, anyhow::Error>(())
363 })
364 .await
365 .context("V21 finalization blocking task panicked")??;
366
367 Ok::<_, anyhow::Error>(())
368 }),
369 );
370
371 handle
372 .await
373 .context("tracked V21 attachment migrator panicked")??;
374
375 tokio::task::spawn_blocking(migration.wrap_blocking(move || prepared.prepare_core_schema()))
380 .await
381 .context("post-V21 schema preparation task panicked")?
382 .context("apply the migrations after the V21 cutover")?;
383 Ok(())
384}
385
386#[cfg(test)]
387#[path = "../tests/support/v20_fixture.rs"]
388mod v20_fixture;
389
390#[cfg(test)]
391mod tests {
392 use khive_db::migrations::ATTACHMENT_CUTOVER_VERSION;
393 use khive_db::stores::blob::FsBlobStore;
394 use khive_storage::BlobStore as _;
395
396 use super::v20_fixture::create_v20_database_fixture;
397 use super::*;
398
399 #[tokio::test]
400 async fn legacy_model_count_includes_recoverable_models_only() {
401 let temp = tempfile::tempdir().unwrap();
402 let db_path = temp.path().join("legacy-model-count.db");
403 let (_, content_ref) = create_v20_database_fixture(&db_path, "moodboard_model", None);
404 let conn = rusqlite::Connection::open(&db_path).unwrap();
405 for (kind, entity_type, deleted_at, content_ref) in [
406 (
407 "artifact",
408 "moodboard_model",
409 Some(2),
410 Some(content_ref.as_str()),
411 ),
412 ("artifact", "visual_asset", None, Some(content_ref.as_str())),
413 ("artifact", "moodboard_model", None, None),
414 (
415 "document",
416 "moodboard_model",
417 None,
418 Some(content_ref.as_str()),
419 ),
420 ] {
421 conn.execute(
422 "INSERT INTO entities (id, namespace, kind, entity_type, name, tags, \
423 created_at, updated_at, deleted_at, content_ref) \
424 VALUES (?1, 'other', ?2, ?3, 'count fixture', '[]', 1, 1, ?4, ?5)",
425 rusqlite::params![
426 uuid::Uuid::new_v4().to_string(),
427 kind,
428 entity_type,
429 deleted_at,
430 content_ref
431 ],
432 )
433 .unwrap();
434 }
435 drop(conn);
436 let backend = StorageBackend::sqlite_for_test(&db_path).unwrap();
437 backend.prepare_core_schema().unwrap();
438 assert_eq!(
439 legacy_preference_model_count(backend.sql().as_ref())
440 .await
441 .unwrap(),
442 2
443 );
444 }
445
446 #[tokio::test]
447 async fn secondary_preflight_counts_across_cutover_states_and_propagates_errors() {
448 let temp = tempfile::tempdir().unwrap();
449 let db_path = temp.path().join("secondary-count.db");
450 let (entity_id, content_ref) = create_v20_database_fixture(&db_path, "visual_asset", None);
451 let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
452 assert_eq!(
453 backend.prepare_core_schema().unwrap(),
454 ATTACHMENT_CUTOVER_VERSION - 1
455 );
456 assert_eq!(
457 backend.attachment_cutover_status().unwrap(),
458 AttachmentCutoverStatus::Pending
459 );
460 assert_eq!(
461 rusqlite::Connection::open(&db_path)
462 .unwrap()
463 .execute("UPDATE entities SET content_ref = NULL", [])
464 .unwrap(),
465 1
466 );
467 require_secondary_attachment_empty(Arc::clone(&backend), "secondary")
468 .await
469 .expect("pending schema has no attachments table");
470
471 let owner = khive_db::stores::blob::acquire_database_gc_owner(backend.sql().as_ref())
472 .await
473 .unwrap();
474 backend.stage_attachment_cutover(&owner).unwrap();
475 assert_eq!(
476 backend.attachment_cutover_status().unwrap(),
477 AttachmentCutoverStatus::Incomplete
478 );
479 require_secondary_attachment_empty(Arc::clone(&backend), "secondary")
480 .await
481 .unwrap();
482 backend.finalize_attachment_cutover(&owner).unwrap();
483 drop(owner);
484 assert_eq!(
485 backend.attachment_cutover_status().unwrap(),
486 AttachmentCutoverStatus::Complete
487 );
488 require_secondary_attachment_empty(Arc::clone(&backend), "secondary")
489 .await
490 .expect("complete schema has no legacy content_ref column");
491
492 rusqlite::Connection::open(&db_path)
493 .unwrap()
494 .execute(
495 "INSERT INTO attachments (record_uuid, substrate, role, content_ref, created_at) \
496 VALUES (?1, 'entity', 'content', ?2, 1)",
497 rusqlite::params![entity_id.to_string(), content_ref.as_str()],
498 )
499 .unwrap();
500 let error = require_secondary_attachment_empty(Arc::clone(&backend), "secondary")
501 .await
502 .expect_err("completed attachment rows still participate in liveness");
503 assert!(error
504 .to_string()
505 .contains("legacy_refs=0, attachment_rows=1"));
506
507 let error = scalar_count(
508 &backend,
509 "SELECT COUNT(*) FROM absent_attachment_count_fixture",
510 "missing_attachment_count",
511 )
512 .await
513 .expect_err("a failed count must never become zero");
514 assert_eq!(error.to_string(), "missing_attachment_count: execute count");
515 assert!(matches!(
516 error.downcast_ref::<khive_storage::StorageError>(),
517 Some(khive_storage::StorageError::Driver { .. })
518 ));
519 }
520
521 #[tokio::test]
522 async fn non_model_legacy_content_resumes_to_attachment_only_v21_without_a_blob_store() {
523 let temp = tempfile::tempdir().unwrap();
524 let db_path = temp.path().join("legacy.db");
525 let (entity_id, content_ref) = create_v20_database_fixture(&db_path, "visual_asset", None);
526 let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
527
528 assert_eq!(
529 backend.prepare_core_schema().unwrap(),
530 ATTACHMENT_CUTOVER_VERSION - 1,
531 "ordinary migration must stop before the coordinated legacy cutover"
532 );
533 coordinate_attachment_cutover(Arc::clone(&backend), None, None)
534 .await
535 .expect("non-model legacy content needs no BlobHydrator");
536 assert_eq!(
537 backend.attachment_cutover_status().unwrap(),
538 AttachmentCutoverStatus::Complete
539 );
540 assert_eq!(
541 backend.schema_version().unwrap(),
542 khive_db::migrations::latest_schema_version(),
543 "the coordinated cutover must leave the schema at the latest migration, \
544 not at V21, before a runtime is built on it"
545 );
546
547 let attachment = backend
548 .attachments()
549 .unwrap()
550 .get_attachment(entity_id, "content")
551 .await
552 .unwrap()
553 .expect("legacy content role must be backfilled");
554 assert_eq!(attachment.content_ref, content_ref);
555
556 let conn = rusqlite::Connection::open(&db_path).unwrap();
557 let legacy_columns: i64 = conn
558 .query_row(
559 "SELECT COUNT(*) FROM pragma_table_info('entities') WHERE name = 'content_ref'",
560 [],
561 |row| row.get(0),
562 )
563 .unwrap();
564 assert_eq!(legacy_columns, 0, "final V21 must drop the legacy column");
565 }
566
567 #[tokio::test]
578 async fn legacy_model_without_verifier_fails_closed_and_gc_refuses_to_run() {
579 let temp = tempfile::tempdir().unwrap();
580 let db_path = temp.path().join("legacy-model.db");
581 create_v20_database_fixture(&db_path, "moodboard_model", None);
582 let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
583 assert_eq!(
584 backend.prepare_core_schema().unwrap(),
585 ATTACHMENT_CUTOVER_VERSION - 1
586 );
587
588 let error = coordinate_attachment_cutover(Arc::clone(&backend), None, None)
589 .await
590 .expect_err("legacy model evidence requires an installed verifier");
591 let message = error.to_string();
592 assert!(
593 message.contains("1 legacy moodboard preference model"),
594 "error must name the legacy count: {message}"
595 );
596 assert!(
597 message.contains("pack-moodboard"),
598 "error must tell the operator which feature to rebuild with: {message}"
599 );
600 assert_eq!(
601 backend.attachment_cutover_status().unwrap(),
602 AttachmentCutoverStatus::Incomplete,
603 "a failed application stage must remain durably resumable"
604 );
605
606 let store = FsBlobStore::new(temp.path().join("blobs"), 0).unwrap();
607 let sweep_error = store
608 .transactional_orphan_sweep(backend.sql().as_ref(), false)
609 .await
610 .expect_err("GC must not run over an incomplete dual representation");
611 assert!(sweep_error.to_string().contains("fencing"));
612 }
613
614 #[tokio::test]
615 async fn secondary_preflight_counts_soft_deleted_legacy_references() {
616 let temp = tempfile::tempdir().unwrap();
617 let db_path = temp.path().join("secondary.db");
618 create_v20_database_fixture(&db_path, "visual_asset", Some(2));
619 let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
620 assert_eq!(
621 backend.prepare_core_schema().unwrap(),
622 ATTACHMENT_CUTOVER_VERSION - 1
623 );
624
625 let error = require_secondary_attachment_empty(backend, "secondary")
626 .await
627 .expect_err("recoverable soft-deleted references still participate in liveness");
628 assert!(error.to_string().contains("legacy_refs=1"));
629 }
630
631 #[tokio::test]
632 #[serial_test::serial]
633 async fn concurrent_boots_recheck_completion_after_waiting_for_gc_ownership() {
634 let temp = tempfile::tempdir().unwrap();
635 let db_path = temp.path().join("concurrent-boot.db");
636 create_v20_database_fixture(&db_path, "visual_asset", None);
637 let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
638 assert_eq!(
639 backend.prepare_core_schema().unwrap(),
640 ATTACHMENT_CUTOVER_VERSION - 1
641 );
642
643 let owner = khive_db::stores::blob::acquire_database_gc_owner(backend.sql().as_ref())
644 .await
645 .unwrap();
646 let barrier = Arc::new(tokio::sync::Barrier::new(3));
647 test_sync::install(
648 std::fs::canonicalize(&db_path).expect("canonical fixture database"),
649 Arc::clone(&barrier),
650 );
651 let first = tokio::spawn(coordinate_attachment_cutover(
652 Arc::clone(&backend),
653 None,
654 None,
655 ));
656 let second = tokio::spawn(coordinate_attachment_cutover(
657 Arc::clone(&backend),
658 None,
659 None,
660 ));
661
662 tokio::time::timeout(std::time::Duration::from_secs(5), barrier.wait())
663 .await
664 .expect("both boot tasks must reach the per-database GC-owner hook");
665 test_sync::clear();
666 drop(owner);
667
668 first.await.unwrap().unwrap();
669 second.await.unwrap().unwrap();
670 assert_eq!(
671 backend.attachment_cutover_status().unwrap(),
672 AttachmentCutoverStatus::Complete
673 );
674 }
675
676 #[tokio::test]
677 async fn interrupted_secondary_revalidates_empty_after_staging() {
678 let temp = tempfile::tempdir().unwrap();
679 let db_path = temp.path().join("secondary-race.db");
680 create_v20_database_fixture(&db_path, "visual_asset", None);
681 let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
682 assert_eq!(
683 backend.prepare_core_schema().unwrap(),
684 ATTACHMENT_CUTOVER_VERSION - 1
685 );
686 let owner = khive_db::stores::blob::acquire_database_gc_owner(backend.sql().as_ref())
687 .await
688 .unwrap();
689 backend.stage_attachment_cutover(&owner).unwrap();
690 drop(owner);
691
692 let error =
693 coordinate_empty_secondary_attachment_cutover(Arc::clone(&backend), "secondary")
694 .await
695 .expect_err("a raced legacy reference must never become secondary liveness");
696 assert!(error.to_string().contains("attachment_rows=1"));
697 assert_eq!(
698 backend.attachment_cutover_status().unwrap(),
699 AttachmentCutoverStatus::Incomplete
700 );
701 }
702}