Skip to main content

khive_runtime/
attachment_cutover.rs

1//! Host-runtime coordination for ADR-121/ADR-160's V21 attachment cutover.
2
3use std::sync::Arc;
4
5use anyhow::Context as _;
6use khive_db::migrations::AttachmentCutoverStatus;
7use khive_storage::types::SqlStatement;
8use khive_storage::{Attachment, AttachmentSubstrate, SqlAccess};
9
10use crate::{BlobHydrator, LegacyPreferenceVerifier, RuntimeError, StorageBackend};
11
12const FANN_NETWORK_ROLE: &str = "fann-network";
13
14/// Count legacy (pre-V21) moodboard preference-model artifact rows,
15/// including soft-deleted ones.
16///
17/// This is the core-side twin of `khive_pack_moodboard::legacy_preference_model_count`
18/// (same SQL, same label) — it lets V21 detect legacy moodboard rows without
19/// linking `khive-pack-moodboard`, which is an optional dependency. The
20/// query itself uses no moodboard types: it is a plain count over `entities`.
21/// `'moodboard_model'` is a value in the data, not a code dependency on the
22/// pack.
23pub async fn legacy_preference_model_count(sql: &dyn SqlAccess) -> Result<u64, RuntimeError> {
24    let mut reader = sql.reader().await?;
25    Ok(reader
26        .count(SqlStatement {
27            sql: "SELECT COUNT(*) FROM entities \
28                  WHERE kind = 'artifact' AND entity_type = 'moodboard_model' \
29                    AND content_ref IS NOT NULL"
30                .to_string(),
31            params: vec![],
32            label: Some("moodboard_legacy_preference_model_count".to_string()),
33        })
34        .await?)
35}
36
37enum CutoverMode {
38    Main,
39    EmptySecondary { backend_name: String },
40}
41
42impl CutoverMode {
43    fn secondary_name(&self) -> Option<&str> {
44        match self {
45            Self::Main => None,
46            Self::EmptySecondary { backend_name } => Some(backend_name),
47        }
48    }
49}
50
51#[cfg(test)]
52mod test_sync {
53    use std::path::PathBuf;
54    use std::sync::{Arc, Mutex, OnceLock};
55
56    use khive_storage::SqlAccess;
57    use tokio::sync::Barrier;
58
59    struct Hook {
60        database_path: PathBuf,
61        barrier: Arc<Barrier>,
62    }
63
64    fn hook() -> &'static Mutex<Option<Hook>> {
65        static HOOK: OnceLock<Mutex<Option<Hook>>> = OnceLock::new();
66        HOOK.get_or_init(|| Mutex::new(None))
67    }
68
69    pub(super) fn install(database_path: PathBuf, barrier: Arc<Barrier>) {
70        *hook().lock().expect("cutover test hook lock") = Some(Hook {
71            database_path,
72            barrier,
73        });
74    }
75
76    pub(super) fn clear() {
77        *hook().lock().expect("cutover test hook lock") = None;
78    }
79
80    pub(super) async fn before_gc_owner(sql: &dyn SqlAccess) {
81        let barrier = hook()
82            .lock()
83            .expect("cutover test hook lock")
84            .as_ref()
85            .filter(|hook| sql.database_path().as_ref() == Some(&hook.database_path))
86            .map(|hook| Arc::clone(&hook.barrier));
87        if let Some(barrier) = barrier {
88            barrier.wait().await;
89        }
90    }
91}
92
93async fn scalar_count(
94    backend: &StorageBackend,
95    sql: &'static str,
96    label: &'static str,
97) -> anyhow::Result<u64> {
98    let sql_access = backend.sql();
99    let mut reader = sql_access
100        .reader()
101        .await
102        .with_context(|| format!("{label}: acquire SQL reader"))?;
103    reader
104        .count(SqlStatement {
105            sql: sql.to_string(),
106            params: vec![],
107            label: Some(label.to_string()),
108        })
109        .await
110        .with_context(|| format!("{label}: execute count"))
111}
112
113async fn cutover_status(backend: Arc<StorageBackend>) -> anyhow::Result<AttachmentCutoverStatus> {
114    tokio::task::spawn_blocking(move || backend.attachment_cutover_status())
115        .await
116        .context("attachment cutover status task panicked")?
117        .context("inspect attachment cutover status")
118}
119
120/// Reject attachment evidence on a non-main database before the main database
121/// exposes its durable incomplete marker.
122///
123/// Phase 4 deliberately chooses one liveness authority: attachment-bearing
124/// records are main-backend-only. This check includes recoverable soft-deleted
125/// legacy entities and already-completed attachment rows.
126pub async fn require_secondary_attachment_empty(
127    backend: Arc<StorageBackend>,
128    backend_name: &str,
129) -> anyhow::Result<()> {
130    let status = cutover_status(Arc::clone(&backend)).await?;
131    let legacy_refs = match status {
132        AttachmentCutoverStatus::Pending | AttachmentCutoverStatus::Incomplete => {
133            scalar_count(
134                backend.as_ref(),
135                "SELECT COUNT(*) FROM entities WHERE content_ref IS NOT NULL",
136                "secondary_legacy_attachment_count",
137            )
138            .await?
139        }
140        AttachmentCutoverStatus::Complete => 0,
141    };
142    let attachment_rows = match status {
143        AttachmentCutoverStatus::Pending => 0,
144        AttachmentCutoverStatus::Incomplete | AttachmentCutoverStatus::Complete => {
145            scalar_count(
146                backend.as_ref(),
147                "SELECT COUNT(*) FROM attachments",
148                "secondary_attachment_row_count",
149            )
150            .await?
151        }
152    };
153
154    if legacy_refs != 0 || attachment_rows != 0 {
155        anyhow::bail!(
156            "backend {backend_name:?} contains attachment-bearing records \
157             (legacy_refs={legacy_refs}, attachment_rows={attachment_rows}); V21 makes the \
158             main backend the sole attachment/GC liveness authority. Move or remove those \
159             recoverable records before retrying boot"
160        );
161    }
162    Ok(())
163}
164
165/// Finish one pending/incomplete database on the real host Tokio runtime.
166///
167/// The outer task is ADR-119 tracked and explicitly awaited. It owns the
168/// canonical DB GC guard across stage, async verified blob hydration, the one
169/// application-attachment transaction, and finalization. Blocking SQLite
170/// phases receive ownership of the guard, so caller cancellation cannot release
171/// it while native work is still running.
172///
173/// This is the public entry point an embedding host calls to bring a V20
174/// database to V21. `KhiveRuntime::new` migrates a writable database to V20
175/// and, when V21 is left to this coordinator, refuses it with
176/// [`RuntimeError::AttachmentCutoverRequired`]; run this function over the
177/// same database, then construct the runtime again.
178///
179/// The cutover stages only from V20. A file below V20 opened directly with a
180/// backend constructor is refused here until
181/// [`StorageBackend::prepare_core_schema`] has migrated it forward (to V20, or
182/// through V21 when no legacy reference needs this coordinator).
183///
184/// `Ok(())` means the V21 cutover is complete. Whether the later migrations
185/// ran depends on the state at entry. When the cutover was already complete,
186/// this returns at once and runs nothing, so the schema can still be behind the
187/// latest version. Otherwise, once the cutover is finished here (or by a
188/// sibling while this call waited for the GC owner), it applies the remaining
189/// migrations before returning. `KhiveRuntime::new` applies them in either
190/// case; `KhiveRuntime::from_prepared_backend` needs
191/// [`StorageBackend::prepare_core_schema`] called first.
192///
193/// A database that holds legacy moodboard preference models needs both a
194/// `verifier` and a `hydrator`; with either one `None`, the call fails closed
195/// and the cutover stays resumable.
196///
197/// # Inspecting the state without migrating
198///
199/// [`StorageBackend::attachment_cutover_status`] is public and only reads: it
200/// takes a pooled reader and inspects the schema bookkeeping, so it can be
201/// called on a pre-V21 file that has been opened with a backend constructor but
202/// not yet prepared. It reports `Pending`, `Incomplete` or `Complete`; it never
203/// runs `prepare_core_schema` and never advances the database.
204///
205/// # Arguments
206///
207/// * `hydrator` - verified blob hydration; required when legacy attachment
208///   evidence exists, otherwise `None` is accepted.
209/// * `verifier` - the host's legacy-preference verifier, or `None` when the
210///   host has none.
211///
212/// # Errors
213///
214/// Returns an error when the status cannot be read, when the database is below
215/// V20 and has not been prepared, when legacy evidence
216/// requires a hydrator that was not configured, or when staging, application or
217/// finalization fails. A failed run leaves the cutover durably resumable.
218pub async fn coordinate_attachment_cutover(
219    backend: Arc<StorageBackend>,
220    hydrator: Option<Arc<BlobHydrator>>,
221    verifier: Option<Arc<dyn LegacyPreferenceVerifier>>,
222) -> anyhow::Result<()> {
223    coordinate_attachment_cutover_inner(backend, hydrator, verifier, CutoverMode::Main).await
224}
225
226/// Finish an interrupted empty secondary without ever making it an
227/// attachment-liveness authority.
228pub async fn coordinate_empty_secondary_attachment_cutover(
229    backend: Arc<StorageBackend>,
230    backend_name: &str,
231) -> anyhow::Result<()> {
232    coordinate_attachment_cutover_inner(
233        backend,
234        None,
235        None,
236        CutoverMode::EmptySecondary {
237            backend_name: backend_name.to_string(),
238        },
239    )
240    .await
241}
242
243async fn coordinate_attachment_cutover_inner(
244    backend: Arc<StorageBackend>,
245    hydrator: Option<Arc<BlobHydrator>>,
246    verifier: Option<Arc<dyn LegacyPreferenceVerifier>>,
247    mode: CutoverMode,
248) -> anyhow::Result<()> {
249    if cutover_status(Arc::clone(&backend)).await? == AttachmentCutoverStatus::Complete {
250        if let Some(backend_name) = mode.secondary_name() {
251            // Exact-current read-only snapshots must remain query-only: GC
252            // ownership creates an advisory lock file, but no mutation or
253            // legacy-column race is possible after the V21 drop. Inventory
254            // the immutable secondary and return without filesystem writes.
255            require_secondary_attachment_empty(backend, backend_name).await?;
256        }
257        return Ok(());
258    }
259
260    let prepared = Arc::clone(&backend);
261    // The tracked child runs on its own task: every poll of it, and every write it
262    // queues, is migration class. Its blocking hops call cutover methods that scope
263    // themselves.
264    let migration = khive_db::WriteClassContext::migration();
265    let handle = crate::spawn_named_tracked_task(
266        "attachment_cutover",
267        migration.scope_future(async move {
268            let sql = backend.sql();
269            #[cfg(test)]
270            test_sync::before_gc_owner(sql.as_ref()).await;
271            let owner = khive_db::stores::blob::acquire_database_gc_owner(sql.as_ref())
272                .await
273                .context("acquire canonical database GC owner for V21")?;
274
275            let (backend, owner, already_complete) = tokio::task::spawn_blocking(move || {
276                // The optimistic check before task creation is not a lock. A
277                // sibling process may complete V21 while this boot waits for the
278                // canonical GC owner, so re-check under ownership before issuing
279                // any SQL that still names the legacy column.
280                if backend.attachment_cutover_status()? == AttachmentCutoverStatus::Complete {
281                    return Ok::<_, anyhow::Error>((backend, owner, true));
282                }
283                backend
284                    .stage_attachment_cutover(&owner)
285                    .context("commit V21 attachment stage 1")?;
286                Ok::<_, anyhow::Error>((backend, owner, false))
287            })
288            .await
289            .context("V21 stage-1 blocking task panicked")??;
290            if already_complete {
291                if let Some(backend_name) = mode.secondary_name() {
292                    require_secondary_attachment_empty(backend, backend_name).await?;
293                }
294                return Ok(());
295            }
296
297            let verified = if let Some(backend_name) = mode.secondary_name() {
298                // Revalidate after stage while the same GC owner remains held.
299                // A legacy writer that raced the earlier inventory is now either
300                // visible both as a legacy ref and a staged content attachment,
301                // or will be caught by finalization's exact backfill recheck.
302                require_secondary_attachment_empty(Arc::clone(&backend), backend_name).await?;
303                Vec::new()
304            } else {
305                let sql = backend.sql();
306                let legacy_model_count = legacy_preference_model_count(sql.as_ref())
307                    .await
308                    .context("count legacy moodboard preference models")?;
309                if legacy_model_count == 0 {
310                    Vec::new()
311                } else {
312                    let verifier = verifier.as_ref().ok_or_else(|| {
313                        anyhow::anyhow!(
314                        "V21 found {legacy_model_count} legacy moodboard preference model(s), but \
315                         this build has no moodboard support (compiled without the \
316                         `pack-moodboard` feature); rebuild with that feature enabled to \
317                         migrate them, then retry boot"
318                    )
319                    })?;
320                    let hydrator = hydrator.as_ref().ok_or_else(|| {
321                        anyhow::anyhow!(
322                        "V21 found {legacy_model_count} legacy moodboard preference model(s), but \
323                         no BlobHydrator is configured; configure [storage.blob] and retry boot"
324                    )
325                    })?;
326                    let verified = verifier
327                        .verify_legacy_preference_attachments(sql.as_ref(), hydrator.as_ref())
328                        .await
329                        .context("verify legacy moodboard bundle/event/FANN evidence")?;
330                    if verified.len() as u64 != legacy_model_count {
331                        anyhow::bail!(
332                            "legacy moodboard verification returned {} attachment(s) for \
333                         {legacy_model_count} recoverable model(s)",
334                            verified.len()
335                        );
336                    }
337                    verified
338                }
339            };
340
341            let created_at = chrono::Utc::now().timestamp_micros();
342            let attachments = verified
343                .into_iter()
344                .map(|verified| Attachment {
345                    record_uuid: verified.model_id,
346                    substrate: AttachmentSubstrate::Entity,
347                    role: FANN_NETWORK_ROLE.to_string(),
348                    content_ref: verified.network_content_ref,
349                    media_type: Some("application/octet-stream".to_string()),
350                    size_bytes: Some(verified.size_bytes),
351                    created_at,
352                })
353                .collect::<Vec<_>>();
354
355            tokio::task::spawn_blocking(move || {
356                backend
357                    .apply_verified_attachments(&owner, &attachments)
358                    .context("atomically apply verified pack-owned attachments")?;
359                backend
360                    .finalize_attachment_cutover(&owner)
361                    .context("finalize V21 attachment/GC cutover")?;
362                Ok::<_, anyhow::Error>(())
363            })
364            .await
365            .context("V21 finalization blocking task panicked")??;
366
367            Ok::<_, anyhow::Error>(())
368        }),
369    );
370
371    handle
372        .await
373        .context("tracked V21 attachment migrator panicked")??;
374
375    // Ordinary schema preparation stops before V21, so a backend whose cutover
376    // was just finished here (or by a sibling while this boot waited for the GC
377    // owner) is still at V21. Apply the remaining migrations before the runtime
378    // is built on it; the GC owner was released when the task above ended.
379    tokio::task::spawn_blocking(migration.wrap_blocking(move || prepared.prepare_core_schema()))
380        .await
381        .context("post-V21 schema preparation task panicked")?
382        .context("apply the migrations after the V21 cutover")?;
383    Ok(())
384}
385
386#[cfg(test)]
387#[path = "../tests/support/v20_fixture.rs"]
388mod v20_fixture;
389
390#[cfg(test)]
391mod tests {
392    use khive_db::migrations::ATTACHMENT_CUTOVER_VERSION;
393    use khive_db::stores::blob::FsBlobStore;
394    use khive_storage::BlobStore as _;
395
396    use super::v20_fixture::create_v20_database_fixture;
397    use super::*;
398
399    #[tokio::test]
400    async fn legacy_model_count_includes_recoverable_models_only() {
401        let temp = tempfile::tempdir().unwrap();
402        let db_path = temp.path().join("legacy-model-count.db");
403        let (_, content_ref) = create_v20_database_fixture(&db_path, "moodboard_model", None);
404        let conn = rusqlite::Connection::open(&db_path).unwrap();
405        for (kind, entity_type, deleted_at, content_ref) in [
406            (
407                "artifact",
408                "moodboard_model",
409                Some(2),
410                Some(content_ref.as_str()),
411            ),
412            ("artifact", "visual_asset", None, Some(content_ref.as_str())),
413            ("artifact", "moodboard_model", None, None),
414            (
415                "document",
416                "moodboard_model",
417                None,
418                Some(content_ref.as_str()),
419            ),
420        ] {
421            conn.execute(
422                "INSERT INTO entities (id, namespace, kind, entity_type, name, tags, \
423                 created_at, updated_at, deleted_at, content_ref) \
424                 VALUES (?1, 'other', ?2, ?3, 'count fixture', '[]', 1, 1, ?4, ?5)",
425                rusqlite::params![
426                    uuid::Uuid::new_v4().to_string(),
427                    kind,
428                    entity_type,
429                    deleted_at,
430                    content_ref
431                ],
432            )
433            .unwrap();
434        }
435        drop(conn);
436        let backend = StorageBackend::sqlite_for_test(&db_path).unwrap();
437        backend.prepare_core_schema().unwrap();
438        assert_eq!(
439            legacy_preference_model_count(backend.sql().as_ref())
440                .await
441                .unwrap(),
442            2
443        );
444    }
445
446    #[tokio::test]
447    async fn secondary_preflight_counts_across_cutover_states_and_propagates_errors() {
448        let temp = tempfile::tempdir().unwrap();
449        let db_path = temp.path().join("secondary-count.db");
450        let (entity_id, content_ref) = create_v20_database_fixture(&db_path, "visual_asset", None);
451        let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
452        assert_eq!(
453            backend.prepare_core_schema().unwrap(),
454            ATTACHMENT_CUTOVER_VERSION - 1
455        );
456        assert_eq!(
457            backend.attachment_cutover_status().unwrap(),
458            AttachmentCutoverStatus::Pending
459        );
460        assert_eq!(
461            rusqlite::Connection::open(&db_path)
462                .unwrap()
463                .execute("UPDATE entities SET content_ref = NULL", [])
464                .unwrap(),
465            1
466        );
467        require_secondary_attachment_empty(Arc::clone(&backend), "secondary")
468            .await
469            .expect("pending schema has no attachments table");
470
471        let owner = khive_db::stores::blob::acquire_database_gc_owner(backend.sql().as_ref())
472            .await
473            .unwrap();
474        backend.stage_attachment_cutover(&owner).unwrap();
475        assert_eq!(
476            backend.attachment_cutover_status().unwrap(),
477            AttachmentCutoverStatus::Incomplete
478        );
479        require_secondary_attachment_empty(Arc::clone(&backend), "secondary")
480            .await
481            .unwrap();
482        backend.finalize_attachment_cutover(&owner).unwrap();
483        drop(owner);
484        assert_eq!(
485            backend.attachment_cutover_status().unwrap(),
486            AttachmentCutoverStatus::Complete
487        );
488        require_secondary_attachment_empty(Arc::clone(&backend), "secondary")
489            .await
490            .expect("complete schema has no legacy content_ref column");
491
492        rusqlite::Connection::open(&db_path)
493            .unwrap()
494            .execute(
495                "INSERT INTO attachments (record_uuid, substrate, role, content_ref, created_at) \
496                 VALUES (?1, 'entity', 'content', ?2, 1)",
497                rusqlite::params![entity_id.to_string(), content_ref.as_str()],
498            )
499            .unwrap();
500        let error = require_secondary_attachment_empty(Arc::clone(&backend), "secondary")
501            .await
502            .expect_err("completed attachment rows still participate in liveness");
503        assert!(error
504            .to_string()
505            .contains("legacy_refs=0, attachment_rows=1"));
506
507        let error = scalar_count(
508            &backend,
509            "SELECT COUNT(*) FROM absent_attachment_count_fixture",
510            "missing_attachment_count",
511        )
512        .await
513        .expect_err("a failed count must never become zero");
514        assert_eq!(error.to_string(), "missing_attachment_count: execute count");
515        assert!(matches!(
516            error.downcast_ref::<khive_storage::StorageError>(),
517            Some(khive_storage::StorageError::Driver { .. })
518        ));
519    }
520
521    #[tokio::test]
522    async fn non_model_legacy_content_resumes_to_attachment_only_v21_without_a_blob_store() {
523        let temp = tempfile::tempdir().unwrap();
524        let db_path = temp.path().join("legacy.db");
525        let (entity_id, content_ref) = create_v20_database_fixture(&db_path, "visual_asset", None);
526        let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
527
528        assert_eq!(
529            backend.prepare_core_schema().unwrap(),
530            ATTACHMENT_CUTOVER_VERSION - 1,
531            "ordinary migration must stop before the coordinated legacy cutover"
532        );
533        coordinate_attachment_cutover(Arc::clone(&backend), None, None)
534            .await
535            .expect("non-model legacy content needs no BlobHydrator");
536        assert_eq!(
537            backend.attachment_cutover_status().unwrap(),
538            AttachmentCutoverStatus::Complete
539        );
540        assert_eq!(
541            backend.schema_version().unwrap(),
542            khive_db::migrations::latest_schema_version(),
543            "the coordinated cutover must leave the schema at the latest migration, \
544             not at V21, before a runtime is built on it"
545        );
546
547        let attachment = backend
548            .attachments()
549            .unwrap()
550            .get_attachment(entity_id, "content")
551            .await
552            .unwrap()
553            .expect("legacy content role must be backfilled");
554        assert_eq!(attachment.content_ref, content_ref);
555
556        let conn = rusqlite::Connection::open(&db_path).unwrap();
557        let legacy_columns: i64 = conn
558            .query_row(
559                "SELECT COUNT(*) FROM pragma_table_info('entities') WHERE name = 'content_ref'",
560                [],
561                |row| row.get(0),
562            )
563            .unwrap();
564        assert_eq!(legacy_columns, 0, "final V21 must drop the legacy column");
565    }
566
567    /// Legacy count > 0 and no verifier installed (the default, feature-off
568    /// build) must fail closed rather than silently dropping the legacy
569    /// column with unmigrated models still on disk.
570    ///
571    /// Was: `legacy_model_without_hydrator_stays_incomplete_and_gc_refuses_to_run`,
572    /// which asserted the error contained "no BlobHydrator is configured".
573    /// Now: no verifier is ever passed in this test, so the *new* fail-closed
574    /// arm fires first — before hydrator configuration is even consulted —
575    /// and the assertion moves to the new message naming the legacy count
576    /// and the missing `pack-moodboard` feature.
577    #[tokio::test]
578    async fn legacy_model_without_verifier_fails_closed_and_gc_refuses_to_run() {
579        let temp = tempfile::tempdir().unwrap();
580        let db_path = temp.path().join("legacy-model.db");
581        create_v20_database_fixture(&db_path, "moodboard_model", None);
582        let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
583        assert_eq!(
584            backend.prepare_core_schema().unwrap(),
585            ATTACHMENT_CUTOVER_VERSION - 1
586        );
587
588        let error = coordinate_attachment_cutover(Arc::clone(&backend), None, None)
589            .await
590            .expect_err("legacy model evidence requires an installed verifier");
591        let message = error.to_string();
592        assert!(
593            message.contains("1 legacy moodboard preference model"),
594            "error must name the legacy count: {message}"
595        );
596        assert!(
597            message.contains("pack-moodboard"),
598            "error must tell the operator which feature to rebuild with: {message}"
599        );
600        assert_eq!(
601            backend.attachment_cutover_status().unwrap(),
602            AttachmentCutoverStatus::Incomplete,
603            "a failed application stage must remain durably resumable"
604        );
605
606        let store = FsBlobStore::new(temp.path().join("blobs"), 0).unwrap();
607        let sweep_error = store
608            .transactional_orphan_sweep(backend.sql().as_ref(), false)
609            .await
610            .expect_err("GC must not run over an incomplete dual representation");
611        assert!(sweep_error.to_string().contains("fencing"));
612    }
613
614    #[tokio::test]
615    async fn secondary_preflight_counts_soft_deleted_legacy_references() {
616        let temp = tempfile::tempdir().unwrap();
617        let db_path = temp.path().join("secondary.db");
618        create_v20_database_fixture(&db_path, "visual_asset", Some(2));
619        let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
620        assert_eq!(
621            backend.prepare_core_schema().unwrap(),
622            ATTACHMENT_CUTOVER_VERSION - 1
623        );
624
625        let error = require_secondary_attachment_empty(backend, "secondary")
626            .await
627            .expect_err("recoverable soft-deleted references still participate in liveness");
628        assert!(error.to_string().contains("legacy_refs=1"));
629    }
630
631    #[tokio::test]
632    #[serial_test::serial]
633    async fn concurrent_boots_recheck_completion_after_waiting_for_gc_ownership() {
634        let temp = tempfile::tempdir().unwrap();
635        let db_path = temp.path().join("concurrent-boot.db");
636        create_v20_database_fixture(&db_path, "visual_asset", None);
637        let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
638        assert_eq!(
639            backend.prepare_core_schema().unwrap(),
640            ATTACHMENT_CUTOVER_VERSION - 1
641        );
642
643        let owner = khive_db::stores::blob::acquire_database_gc_owner(backend.sql().as_ref())
644            .await
645            .unwrap();
646        let barrier = Arc::new(tokio::sync::Barrier::new(3));
647        test_sync::install(
648            std::fs::canonicalize(&db_path).expect("canonical fixture database"),
649            Arc::clone(&barrier),
650        );
651        let first = tokio::spawn(coordinate_attachment_cutover(
652            Arc::clone(&backend),
653            None,
654            None,
655        ));
656        let second = tokio::spawn(coordinate_attachment_cutover(
657            Arc::clone(&backend),
658            None,
659            None,
660        ));
661
662        tokio::time::timeout(std::time::Duration::from_secs(5), barrier.wait())
663            .await
664            .expect("both boot tasks must reach the per-database GC-owner hook");
665        test_sync::clear();
666        drop(owner);
667
668        first.await.unwrap().unwrap();
669        second.await.unwrap().unwrap();
670        assert_eq!(
671            backend.attachment_cutover_status().unwrap(),
672            AttachmentCutoverStatus::Complete
673        );
674    }
675
676    #[tokio::test]
677    async fn interrupted_secondary_revalidates_empty_after_staging() {
678        let temp = tempfile::tempdir().unwrap();
679        let db_path = temp.path().join("secondary-race.db");
680        create_v20_database_fixture(&db_path, "visual_asset", None);
681        let backend = Arc::new(StorageBackend::sqlite_for_test(&db_path).unwrap());
682        assert_eq!(
683            backend.prepare_core_schema().unwrap(),
684            ATTACHMENT_CUTOVER_VERSION - 1
685        );
686        let owner = khive_db::stores::blob::acquire_database_gc_owner(backend.sql().as_ref())
687            .await
688            .unwrap();
689        backend.stage_attachment_cutover(&owner).unwrap();
690        drop(owner);
691
692        let error =
693            coordinate_empty_secondary_attachment_cutover(Arc::clone(&backend), "secondary")
694                .await
695                .expect_err("a raced legacy reference must never become secondary liveness");
696        assert!(error.to_string().contains("attachment_rows=1"));
697        assert_eq!(
698            backend.attachment_cutover_status().unwrap(),
699            AttachmentCutoverStatus::Incomplete
700        );
701    }
702}