Skip to main content

khive_runtime/
blob.rs

1//! Config-driven `BlobStore` selection (ADR-111 Amendment 2).
2//!
3//! `khive-db` cannot parse `khive.toml` itself (it sits below `khive-runtime`
4//! in the crate dependency chain), so the fs-vs-s3 selector lives here, one
5//! layer up, where `KhiveConfig` is already parsed. This is the choke point
6//! every boot path (single- and multi-backend) resolves the configured blob
7//! store through, so the two never drift onto different construction logic.
8
9use std::sync::Arc;
10
11use async_trait::async_trait;
12use khive_db::stores::blob_s3::{S3BlobStore, S3BlobStoreConfig};
13use khive_db::{SqliteError, StorageBackend};
14use khive_storage::{
15    BlobOrphanSweepConfig, BlobOrphanSweepResult, BlobStore, ContentRef, SqlAccess,
16    StorageCapability, StorageError, StorageResult,
17};
18
19use crate::engine_config::BlobConfig;
20use crate::{KhiveConfig, RuntimeError, RuntimeResult};
21
22/// Default process-local admission budget for resident verified blob buffers.
23pub const DEFAULT_BLOB_HYDRATION_BYTES: u64 = 4 * khive_storage::MAX_BLOB_WHOLE_BYTES;
24
25/// Runtime-owned bounded blob hydration with weighted raw-byte admission.
26#[derive(Debug)]
27pub struct BlobHydrator {
28    store: Arc<dyn BlobStore>,
29    /// The store as handed in by the caller, before any read-only wrapping.
30    /// Kept so the install seam can recognize a reinstall of the same raw
31    /// store by pointer identity even when `store` is a wrapper around it.
32    raw_store: Arc<dyn BlobStore>,
33    /// True only for hydrators built through [`Self::new_read_only`], whose
34    /// mutators are refused by construction. The runtime's install seam
35    /// requires this on read-only runtimes: `dyn BlobStore` carries no
36    /// downcast hook, so the wrapper cannot be recognized after the fact —
37    /// the guarantee has to travel with the hydrator that made it.
38    read_only: bool,
39    /// True only for hydrators built by
40    /// [`Self::resolve_for_governing_backend`], whose mode was derived from a
41    /// governing backend's own access mode rather than declared by the
42    /// caller. The shared install seam accepts only governed hydrators, so a
43    /// hand-paired writable hydrator cannot ride it onto a read-only
44    /// runtime.
45    governed: bool,
46    admission: Arc<tokio::sync::Semaphore>,
47    budget_bytes: u64,
48}
49
50/// Error from [`BlobHydrator::resolve_for_governing_backend`], split so boot
51/// can distinguish store resolution failures (which fall back to "no blob
52/// configured" when `[storage.blob]` is absent) from budget pairing failures
53/// (always fatal).
54#[derive(Debug)]
55pub enum GovernedBlobError {
56    /// The configured store could not be resolved for the governing mode.
57    Resolve(SqliteError),
58    /// The resolved store could not be paired with the hydration budget.
59    Construct(RuntimeError),
60}
61
62impl std::fmt::Display for GovernedBlobError {
63    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
64        match self {
65            Self::Resolve(error) => write!(f, "blob store resolution: {error}"),
66            Self::Construct(error) => write!(f, "blob hydrator construction: {error}"),
67        }
68    }
69}
70
71impl std::error::Error for GovernedBlobError {}
72
73impl BlobHydrator {
74    /// Pair the raw store with a budget, refusing every physical mutator:
75    /// the store is wrapped so `put`/`delete`/sweeps error while the bounded
76    /// read surface stays available. This is the only constructor a
77    /// read-only runtime's install seam accepts.
78    pub(crate) fn new_read_only(
79        store: Arc<dyn BlobStore>,
80        budget_bytes: u64,
81    ) -> RuntimeResult<Self> {
82        let mut hydrator =
83            Self::new_inner(wrap_read_only(Arc::clone(&store)), store, budget_bytes)?;
84        hydrator.read_only = true;
85        Ok(hydrator)
86    }
87
88    /// Whether this hydrator refuses mutation by construction.
89    pub(crate) fn enforces_read_only(&self) -> bool {
90        self.read_only
91    }
92
93    /// The store exactly as the caller handed it in, before any wrapping.
94    pub(crate) fn raw_store(&self) -> Arc<dyn BlobStore> {
95        Arc::clone(&self.raw_store)
96    }
97
98    /// Pair a raw store with a budget under an explicitly declared
99    /// blob-runtime mode. `read_only = true` wraps the store so every
100    /// physical mutator refuses while the bounded read surface stays
101    /// available; `false` is [`Self::new`]. Boot paths that decide the blob
102    /// mode from configuration (the blob pack's backend mode, ADR-160 D3)
103    /// construct through this so the decision travels with the hydrator —
104    /// the install seam can then hold hydrator mode against runtime mode
105    /// instead of trusting the caller's pairing.
106    pub fn for_mode(
107        store: Arc<dyn BlobStore>,
108        budget_bytes: u64,
109        read_only: bool,
110    ) -> RuntimeResult<Self> {
111        if read_only {
112            Self::new_read_only(store, budget_bytes)
113        } else {
114            Self::new(store, budget_bytes)
115        }
116    }
117
118    /// Resolve the configured store and pair it with the budget under the
119    /// mode of the backend that GOVERNS blob mutability — the backend the
120    /// blob pack maps to (ADR-160 D3), which on single-backend boots is the
121    /// runtime's own backend.
122    ///
123    /// The mode is derived here from `governing_backend`'s own access mode,
124    /// never accepted as a caller-declared flag, and the hydrator is stamped
125    /// as governed. [`crate::KhiveRuntime::install_shared_blob_hydrator`]
126    /// accepts only governed hydrators.
127    ///
128    /// Trust model (explicit policy): which backend governs blob mutability
129    /// is a deployment-topology assertion made by the host that wires boot
130    /// (the blob pack's backend, ADR-160 D3), and this seam takes the
131    /// caller's word for it. What the derivation defends against is the
132    /// ACCIDENTAL mode mismatch — a hand-paired writable hydrator drifting
133    /// onto a read-only handle through the shared seam. It does not defend
134    /// against an in-process caller who deliberately misdeclares the
135    /// governing backend, because no runtime seam can: any such caller can
136    /// already open the configured blob root directly (the same config and
137    /// store constructors are public) and mutate it without touching a
138    /// runtime handle. Read-only runtime handles are a wrong-wiring guard,
139    /// not an in-process sandbox.
140    pub fn resolve_for_governing_backend(
141        cfg: &KhiveConfig,
142        resolve_backend: &StorageBackend,
143        governing_backend: &StorageBackend,
144        budget_bytes: u64,
145    ) -> Result<Self, GovernedBlobError> {
146        let read_only = governing_backend.is_read_only();
147        let store = resolve_blob_store_for_mode(cfg, resolve_backend, read_only)
148            .map_err(GovernedBlobError::Resolve)?;
149        let mut hydrator =
150            Self::for_mode(store, budget_bytes, read_only).map_err(GovernedBlobError::Construct)?;
151        hydrator.governed = true;
152        Ok(hydrator)
153    }
154
155    /// Whether this hydrator's mode was derived from a governing backend
156    /// (see [`Self::resolve_for_governing_backend`]).
157    pub(crate) fn is_governed(&self) -> bool {
158        self.governed
159    }
160
161    /// Pair one store with one aggregate byte budget.
162    pub fn new(store: Arc<dyn BlobStore>, budget_bytes: u64) -> RuntimeResult<Self> {
163        let raw = Arc::clone(&store);
164        Self::new_inner(store, raw, budget_bytes)
165    }
166
167    fn new_inner(
168        store: Arc<dyn BlobStore>,
169        raw_store: Arc<dyn BlobStore>,
170        budget_bytes: u64,
171    ) -> RuntimeResult<Self> {
172        if budget_bytes < khive_storage::MAX_BLOB_WHOLE_BYTES {
173            return Err(RuntimeError::InvalidInput(format!(
174                "blob hydration budget must be at least {} bytes, got {budget_bytes}",
175                khive_storage::MAX_BLOB_WHOLE_BYTES
176            )));
177        }
178        let permits = usize::try_from(budget_bytes).map_err(|_| {
179            RuntimeError::InvalidInput(format!(
180                "blob hydration budget {budget_bytes} does not fit this platform"
181            ))
182        })?;
183        if permits > tokio::sync::Semaphore::MAX_PERMITS {
184            return Err(RuntimeError::InvalidInput(format!(
185                "blob hydration budget {budget_bytes} exceeds the runtime maximum {}",
186                tokio::sync::Semaphore::MAX_PERMITS
187            )));
188        }
189        Ok(Self {
190            store,
191            raw_store,
192            read_only: false,
193            governed: false,
194            admission: Arc::new(tokio::sync::Semaphore::new(permits)),
195            budget_bytes,
196        })
197    }
198
199    /// Return the resolved aggregate admission budget.
200    pub fn budget_bytes(&self) -> u64 {
201        self.budget_bytes
202    }
203
204    /// Clone the paired store for metadata and mutation paths.
205    ///
206    /// Whole-buffer production reads must use [`Self::hydrate_verified`].
207    pub(crate) fn store(&self) -> Arc<dyn BlobStore> {
208        Arc::clone(&self.store)
209    }
210
211    /// Hydrate one complete digest-verified object under weighted admission.
212    pub async fn hydrate_verified(
213        &self,
214        content_ref: &ContentRef,
215        max_bytes: u64,
216    ) -> RuntimeResult<VerifiedBlob> {
217        if max_bytes > khive_storage::MAX_BLOB_WHOLE_BYTES {
218            return Err(RuntimeError::InvalidInput(format!(
219                "blob hydration max_bytes must not exceed {} bytes, got {max_bytes}",
220                khive_storage::MAX_BLOB_WHOLE_BYTES
221            )));
222        }
223
224        let acquire = Arc::clone(&self.admission).acquire_many_owned(max_bytes as u32);
225        let admission =
226            khive_storage::await_request_read_phase("blob_hydration_admission", acquire)
227                .await?
228                .map_err(|error| {
229                    RuntimeError::Internal(format!("blob hydration admission closed: {error}"))
230                })?;
231
232        let (sender, receiver) = tokio::sync::oneshot::channel::<RuntimeResult<VerifiedBlob>>();
233        let store = Arc::clone(&self.store);
234        let content_ref = content_ref.clone();
235        crate::track_background_task(async move {
236            // The tracked supervisor itself owns backend work and the lease.
237            // Dropping a request only drops `receiver`; the supervisor stays
238            // visible to daemon drain until the backend future actually ends.
239            let result = match store.get_bounded_verified(&content_ref, max_bytes).await {
240                Ok(bytes) => Ok(VerifiedBlob {
241                    bytes,
242                    _admission: admission,
243                }),
244                Err(error) => Err(RuntimeError::Storage(error)),
245            };
246            let _ = sender.send(result);
247        });
248
249        khive_storage::await_request_read_phase("blob_hydration", receiver)
250            .await?
251            .map_err(|_| {
252                RuntimeError::Internal(
253                    "blob hydration supervisor ended without delivering a result".to_string(),
254                )
255            })?
256    }
257}
258
259/// One verified raw blob buffer and its aggregate admission lease.
260///
261/// The wrapper intentionally has no `Clone` or owned-byte extraction API.
262/// Borrowed bytes may still be copied under the caller's own allocation budget.
263///
264/// ```compile_fail
265/// use khive_runtime::VerifiedBlob;
266///
267/// fn clone_verified(blob: &VerifiedBlob) {
268///     let _ = <VerifiedBlob as Clone>::clone(blob);
269/// }
270/// ```
271///
272/// ```compile_fail
273/// use khive_runtime::VerifiedBlob;
274///
275/// fn extract_owned(blob: VerifiedBlob) -> Vec<u8> {
276///     blob.into_bytes()
277/// }
278/// ```
279///
280/// ```compile_fail
281/// use khive_runtime::VerifiedBlob;
282///
283/// fn access_private_storage(blob: &VerifiedBlob) -> usize {
284///     blob.bytes.len()
285/// }
286/// ```
287pub struct VerifiedBlob {
288    bytes: Vec<u8>,
289    _admission: tokio::sync::OwnedSemaphorePermit,
290}
291
292impl VerifiedBlob {
293    /// Borrow the verified bytes without releasing weighted admission.
294    pub fn bytes(&self) -> &[u8] {
295        &self.bytes
296    }
297}
298
299impl AsRef<[u8]> for VerifiedBlob {
300    fn as_ref(&self) -> &[u8] {
301        self.bytes()
302    }
303}
304
305impl std::fmt::Debug for VerifiedBlob {
306    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
307        formatter
308            .debug_struct("VerifiedBlob")
309            .field("len", &self.bytes.len())
310            .finish_non_exhaustive()
311    }
312}
313
314/// Resolve the `BlobStore` this `backend` should use, per `cfg.storage.blob`.
315///
316/// - Absent, or `backend = "fs"`: `FsBlobStore` via `StorageBackend::blob_store`,
317///   using the existing `KHIVE_BLOB_ROOT` > `root` > `<db_dir>/blobs` precedence
318///   (khive#292) — unchanged from every configuration written before this
319///   section existed.
320/// - `backend = "s3"`: `S3BlobStore`, built from the non-secret TOML fields
321///   plus environment credentials (`S3BlobStore::new`).
322pub fn resolve_blob_store(
323    cfg: &KhiveConfig,
324    backend: &StorageBackend,
325) -> Result<Arc<dyn BlobStore>, SqliteError> {
326    match &cfg.storage.blob {
327        None => backend.blob_store(None, None),
328        Some(BlobConfig::Fs { root, floor_bytes }) => {
329            let root_path = root.as_ref().map(std::path::PathBuf::from);
330            backend.blob_store(root_path.as_deref(), *floor_bytes)
331        }
332        Some(BlobConfig::S3 {
333            bucket,
334            region,
335            endpoint,
336            prefix,
337            allow_http,
338        }) => {
339            let mut s3_cfg = S3BlobStoreConfig::new(bucket.clone(), region.clone());
340            if let Some(endpoint) = endpoint {
341                s3_cfg = s3_cfg.with_endpoint(endpoint.clone());
342            }
343            if let Some(prefix) = prefix {
344                s3_cfg = s3_cfg.with_prefix(prefix.clone());
345            }
346            if let Some(allow_http) = allow_http {
347                s3_cfg = s3_cfg.with_allow_http(*allow_http);
348            }
349            let store = S3BlobStore::new(s3_cfg)?;
350            Ok(Arc::new(store))
351        }
352    }
353}
354
355/// Resolve the configured store for one pack runtime's effective access mode.
356///
357/// A read-only runtime retains bounded verified reads plus `exists`/`size`
358/// against an already-present fs root (or a configured S3 store), but boot
359/// never creates the default fs root and the wrapper rejects every physical
360/// mutator. The mode belongs to the runtime assigned to the `blob` pack; a
361/// mixed topology must not infer it from the main audit backend.
362pub fn resolve_blob_store_for_mode(
363    cfg: &KhiveConfig,
364    backend: &StorageBackend,
365    read_only: bool,
366) -> Result<Arc<dyn BlobStore>, SqliteError> {
367    if !read_only {
368        return resolve_blob_store(cfg, backend);
369    }
370
371    let inner: Arc<dyn BlobStore> = match &cfg.storage.blob {
372        None => backend.blob_store_read_only(None, None)?,
373        Some(BlobConfig::Fs { root, floor_bytes }) => {
374            let root_path = root.as_ref().map(std::path::PathBuf::from);
375            backend.blob_store_read_only(root_path.as_deref(), *floor_bytes)?
376        }
377        Some(BlobConfig::S3 {
378            bucket,
379            region,
380            endpoint,
381            prefix,
382            allow_http,
383        }) => {
384            let mut s3_cfg = S3BlobStoreConfig::new(bucket.clone(), region.clone());
385            if let Some(endpoint) = endpoint {
386                s3_cfg = s3_cfg.with_endpoint(endpoint.clone());
387            }
388            if let Some(prefix) = prefix {
389                s3_cfg = s3_cfg.with_prefix(prefix.clone());
390            }
391            if let Some(allow_http) = allow_http {
392                s3_cfg = s3_cfg.with_allow_http(*allow_http);
393            }
394            Arc::new(S3BlobStore::new(s3_cfg)?)
395        }
396    };
397    Ok(Arc::new(ReadOnlyBlobStore { inner }))
398}
399
400/// Wrap an arbitrary store so every physical mutator is refused while the
401/// bounded read surface stays available. Used by the runtime's install seam
402/// to hold the read-only invariant for stores installed after boot; wrapping
403/// an already-wrapped store is harmless (reads delegate, mutators refuse at
404/// the outer layer).
405pub(crate) fn wrap_read_only(inner: Arc<dyn BlobStore>) -> Arc<dyn BlobStore> {
406    Arc::new(ReadOnlyBlobStore { inner })
407}
408
409#[derive(Debug)]
410struct ReadOnlyBlobStore {
411    inner: Arc<dyn BlobStore>,
412}
413
414impl ReadOnlyBlobStore {
415    fn mutation_error(operation: &'static str) -> StorageError {
416        StorageError::Unsupported {
417            capability: StorageCapability::Blob,
418            operation: operation.into(),
419            message: "blob storage is read-only for this pack runtime".to_string(),
420        }
421    }
422}
423
424#[async_trait]
425impl BlobStore for ReadOnlyBlobStore {
426    async fn put(&self, _bytes: Vec<u8>) -> StorageResult<ContentRef> {
427        Err(Self::mutation_error("put"))
428    }
429
430    async fn get_bounded_verified(
431        &self,
432        content_ref: &ContentRef,
433        max_bytes: u64,
434    ) -> StorageResult<Vec<u8>> {
435        self.inner
436            .get_bounded_verified(content_ref, max_bytes)
437            .await
438    }
439
440    async fn exists(&self, content_ref: &ContentRef) -> StorageResult<bool> {
441        self.inner.exists(content_ref).await
442    }
443
444    async fn size(&self, content_ref: &ContentRef) -> StorageResult<Option<u64>> {
445        self.inner.size(content_ref).await
446    }
447
448    async fn delete(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
449        Err(Self::mutation_error("delete"))
450    }
451
452    async fn orphan_sweep(
453        &self,
454        _config: &BlobOrphanSweepConfig,
455    ) -> StorageResult<BlobOrphanSweepResult> {
456        Err(Self::mutation_error("orphan_sweep"))
457    }
458
459    async fn transactional_orphan_sweep(
460        &self,
461        _sql: &dyn SqlAccess,
462        _dry_run: bool,
463    ) -> StorageResult<BlobOrphanSweepResult> {
464        Err(Self::mutation_error("transactional_orphan_sweep"))
465    }
466}
467
468#[cfg(test)]
469mod tests {
470    use super::*;
471    use crate::engine_config::StorageSectionConfig;
472    use serial_test::serial;
473
474    #[derive(Debug, Default)]
475    struct RecordingReadStore {
476        bounded_call: std::sync::Mutex<Option<(ContentRef, u64)>>,
477    }
478
479    #[async_trait]
480    impl BlobStore for RecordingReadStore {
481        async fn put(&self, _bytes: Vec<u8>) -> StorageResult<ContentRef> {
482            panic!("put is not used by the read-only delegation test")
483        }
484
485        async fn get_bounded_verified(
486            &self,
487            content_ref: &ContentRef,
488            max_bytes: u64,
489        ) -> StorageResult<Vec<u8>> {
490            *self.bounded_call.lock().unwrap() = Some((content_ref.clone(), max_bytes));
491            Ok(b"verified".to_vec())
492        }
493
494        async fn exists(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
495            Ok(true)
496        }
497
498        async fn size(&self, _content_ref: &ContentRef) -> StorageResult<Option<u64>> {
499            Ok(Some(8))
500        }
501
502        async fn delete(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
503            panic!("delete is not used by the read-only delegation test")
504        }
505    }
506
507    fn memory_backend() -> StorageBackend {
508        StorageBackend::memory().expect("memory backend should create")
509    }
510
511    #[tokio::test]
512    async fn read_only_store_delegates_the_bounded_verified_read_unchanged() {
513        let inner = Arc::new(RecordingReadStore::default());
514        let store = ReadOnlyBlobStore {
515            inner: Arc::clone(&inner) as Arc<dyn BlobStore>,
516        };
517        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
518
519        let bytes = store.get_bounded_verified(&content_ref, 17).await.unwrap();
520        assert_eq!(bytes, b"verified");
521        assert_eq!(*inner.bounded_call.lock().unwrap(), Some((content_ref, 17)));
522    }
523
524    #[derive(Debug)]
525    struct HydrationReadStore {
526        calls: std::sync::atomic::AtomicUsize,
527        bytes: Vec<u8>,
528        first_started: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
529        first_release: Option<Arc<tokio::sync::Semaphore>>,
530        fail_first: bool,
531        panic_first: bool,
532    }
533
534    impl HydrationReadStore {
535        fn immediate(bytes: Vec<u8>) -> Arc<Self> {
536            Arc::new(Self {
537                calls: std::sync::atomic::AtomicUsize::new(0),
538                bytes,
539                first_started: std::sync::Mutex::new(None),
540                first_release: None,
541                fail_first: false,
542                panic_first: false,
543            })
544        }
545
546        fn blocking_first(
547            bytes: Vec<u8>,
548        ) -> (
549            Arc<Self>,
550            tokio::sync::oneshot::Receiver<()>,
551            Arc<tokio::sync::Semaphore>,
552        ) {
553            let (started_tx, started_rx) = tokio::sync::oneshot::channel();
554            let release = Arc::new(tokio::sync::Semaphore::new(0));
555            (
556                Arc::new(Self {
557                    calls: std::sync::atomic::AtomicUsize::new(0),
558                    bytes,
559                    first_started: std::sync::Mutex::new(Some(started_tx)),
560                    first_release: Some(Arc::clone(&release)),
561                    fail_first: false,
562                    panic_first: false,
563                }),
564                started_rx,
565                release,
566            )
567        }
568
569        fn blocking_first_panic(
570            bytes: Vec<u8>,
571        ) -> (
572            Arc<Self>,
573            tokio::sync::oneshot::Receiver<()>,
574            Arc<tokio::sync::Semaphore>,
575        ) {
576            let (store, started, release) = Self::blocking_first(bytes);
577            let store = Arc::try_unwrap(store).expect("new fixture has one owner");
578            (
579                Arc::new(Self {
580                    panic_first: true,
581                    ..store
582                }),
583                started,
584                release,
585            )
586        }
587
588        fn blocking_first_error(
589            bytes: Vec<u8>,
590        ) -> (
591            Arc<Self>,
592            tokio::sync::oneshot::Receiver<()>,
593            Arc<tokio::sync::Semaphore>,
594        ) {
595            let (store, started, release) = Self::blocking_first(bytes);
596            let store = Arc::try_unwrap(store).expect("new fixture has one owner");
597            (
598                Arc::new(Self {
599                    fail_first: true,
600                    ..store
601                }),
602                started,
603                release,
604            )
605        }
606
607        fn calls(&self) -> usize {
608            self.calls.load(std::sync::atomic::Ordering::SeqCst)
609        }
610    }
611
612    #[async_trait]
613    impl BlobStore for HydrationReadStore {
614        async fn put(&self, _bytes: Vec<u8>) -> StorageResult<ContentRef> {
615            panic!("put is not used by hydrator tests")
616        }
617
618        async fn get_bounded_verified(
619            &self,
620            content_ref: &ContentRef,
621            _max_bytes: u64,
622        ) -> StorageResult<Vec<u8>> {
623            let call = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
624            if call == 0 {
625                if let Some(started) = self.first_started.lock().unwrap().take() {
626                    let _ = started.send(());
627                }
628                if let Some(release) = &self.first_release {
629                    release
630                        .clone()
631                        .acquire_owned()
632                        .await
633                        .expect("test release semaphore stays open")
634                        .forget();
635                }
636                if self.fail_first {
637                    return Err(StorageError::BlobDigestMismatch {
638                        expected: content_ref.clone(),
639                        actual: ContentRef::from_hex("b".repeat(64))
640                            .expect("fixture digest is canonical"),
641                    });
642                }
643                assert!(!self.panic_first, "injected hydration backend panic");
644            }
645            Ok(self.bytes.clone())
646        }
647
648        async fn exists(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
649            panic!("exists is not used by hydrator tests")
650        }
651
652        async fn size(&self, _content_ref: &ContentRef) -> StorageResult<Option<u64>> {
653            panic!("size is not used by hydrator tests")
654        }
655
656        async fn delete(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
657            panic!("delete is not used by hydrator tests")
658        }
659    }
660
661    async fn wait_for_hydration_calls(store: &HydrationReadStore, expected: usize) {
662        tokio::time::timeout(std::time::Duration::from_secs(1), async {
663            while store.calls() != expected {
664                tokio::task::yield_now().await;
665            }
666        })
667        .await
668        .expect("hydration backend call count should advance");
669    }
670
671    async fn wait_for_background_task_count(expected: usize) {
672        for _ in 0..10_000 {
673            if crate::background_task_count() == expected {
674                return;
675            }
676            tokio::task::yield_now().await;
677        }
678        assert_eq!(crate::background_task_count(), expected);
679    }
680
681    #[tokio::test]
682    async fn hydrator_rejects_an_invalid_maximum_before_admission_or_backend_work() {
683        let store = HydrationReadStore::immediate(b"x".to_vec());
684        let hydrator = BlobHydrator::new(
685            Arc::clone(&store) as Arc<dyn BlobStore>,
686            khive_storage::MAX_BLOB_WHOLE_BYTES,
687        )
688        .unwrap();
689        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
690
691        let error = hydrator
692            .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES + 1)
693            .await
694            .unwrap_err();
695
696        assert!(matches!(error, crate::RuntimeError::InvalidInput(_)));
697        assert_eq!(store.calls(), 0);
698    }
699
700    #[tokio::test]
701    #[serial(background_tasks)]
702    async fn zero_maximum_and_idle_portable_maximum_are_both_admissible() {
703        let before = crate::background_task_count();
704        let empty_store = HydrationReadStore::immediate(Vec::new());
705        let empty_hydrator = BlobHydrator::new(
706            Arc::clone(&empty_store) as Arc<dyn BlobStore>,
707            khive_storage::MAX_BLOB_WHOLE_BYTES,
708        )
709        .unwrap();
710        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
711
712        let empty = empty_hydrator
713            .hydrate_verified(&content_ref, 0)
714            .await
715            .expect("zero is a valid declared maximum");
716        assert!(empty.bytes().is_empty());
717        drop(empty);
718
719        let max_store = HydrationReadStore::immediate(b"bounded".to_vec());
720        let max_hydrator = BlobHydrator::new(
721            Arc::clone(&max_store) as Arc<dyn BlobStore>,
722            khive_storage::MAX_BLOB_WHOLE_BYTES,
723        )
724        .unwrap();
725        let max = max_hydrator
726            .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
727            .await
728            .expect("an idle minimum-size budget admits every valid maximum");
729        assert_eq!(max.bytes(), b"bounded");
730        drop(max);
731        wait_for_background_task_count(before).await;
732    }
733
734    #[tokio::test]
735    #[serial(background_tasks)]
736    async fn cancelling_a_waiter_retains_admission_until_backend_completion() {
737        let before = crate::background_task_count();
738        let (store, first_started, first_release) =
739            HydrationReadStore::blocking_first(b"x".to_vec());
740        let hydrator = Arc::new(
741            BlobHydrator::new(
742                Arc::clone(&store) as Arc<dyn BlobStore>,
743                khive_storage::MAX_BLOB_WHOLE_BYTES,
744            )
745            .unwrap(),
746        );
747        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
748
749        let first_hydrator = Arc::clone(&hydrator);
750        let first_ref = content_ref.clone();
751        let first = tokio::spawn(async move {
752            first_hydrator
753                .hydrate_verified(&first_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
754                .await
755        });
756        first_started.await.expect("first backend read must start");
757        assert_eq!(crate::background_task_count(), before + 1);
758
759        first.abort();
760        assert!(first.await.unwrap_err().is_cancelled());
761
762        let second_hydrator = Arc::clone(&hydrator);
763        let second_ref = content_ref.clone();
764        let second =
765            tokio::spawn(async move { second_hydrator.hydrate_verified(&second_ref, 1).await });
766        assert_eq!(hydrator.admission.available_permits(), 0);
767        assert_eq!(
768            store.calls(),
769            1,
770            "cancelled waiters must not release admission held by native work"
771        );
772
773        first_release.add_permits(1);
774        let verified = tokio::time::timeout(std::time::Duration::from_secs(1), second)
775            .await
776            .expect("second waiter should acquire after native completion")
777            .expect("second waiter task should not panic")
778            .expect("second hydration should succeed");
779        assert_eq!(verified.bytes(), b"x");
780        drop(verified);
781        wait_for_hydration_calls(&store, 2).await;
782
783        wait_for_background_task_count(before).await;
784    }
785
786    #[tokio::test]
787    #[serial(background_tasks)]
788    async fn verified_blob_retains_its_weighted_lease_until_drop() {
789        let before = crate::background_task_count();
790        let store = HydrationReadStore::immediate(b"x".to_vec());
791        let hydrator = Arc::new(
792            BlobHydrator::new(
793                Arc::clone(&store) as Arc<dyn BlobStore>,
794                khive_storage::MAX_BLOB_WHOLE_BYTES,
795            )
796            .unwrap(),
797        );
798        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
799
800        let first = hydrator
801            .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
802            .await
803            .unwrap();
804        assert_eq!(first.bytes(), b"x");
805
806        let second_hydrator = Arc::clone(&hydrator);
807        let second_ref = content_ref.clone();
808        let second =
809            tokio::spawn(async move { second_hydrator.hydrate_verified(&second_ref, 1).await });
810        assert_eq!(hydrator.admission.available_permits(), 0);
811        assert_eq!(
812            store.calls(),
813            1,
814            "borrowed access must not release the original raw-buffer lease"
815        );
816
817        drop(first);
818        let second = tokio::time::timeout(std::time::Duration::from_secs(1), second)
819            .await
820            .expect("dropping VerifiedBlob should release its lease")
821            .expect("second waiter task should not panic")
822            .expect("second hydration should succeed");
823        assert_eq!(second.bytes(), b"x");
824        drop(second);
825        wait_for_background_task_count(before).await;
826    }
827
828    #[tokio::test]
829    #[serial(background_tasks)]
830    async fn cancelling_while_queued_starts_no_backend_work() {
831        let before = crate::background_task_count();
832        let store = HydrationReadStore::immediate(b"x".to_vec());
833        let hydrator = Arc::new(
834            BlobHydrator::new(
835                Arc::clone(&store) as Arc<dyn BlobStore>,
836                khive_storage::MAX_BLOB_WHOLE_BYTES,
837            )
838            .unwrap(),
839        );
840        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
841        let first = hydrator
842            .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
843            .await
844            .unwrap();
845
846        let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false);
847        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
848        let queued_hydrator = Arc::clone(&hydrator);
849        let queued_ref = content_ref.clone();
850        let queued = tokio::spawn(async move {
851            khive_storage::scope_request_read_cancellation(cancel_rx, async move {
852                let _ = entered_tx.send(());
853                queued_hydrator.hydrate_verified(&queued_ref, 1).await
854            })
855            .await
856        });
857        entered_rx.await.expect("queued request must be polled");
858        cancel_tx.send(true).unwrap();
859
860        let error = queued.await.unwrap().unwrap_err();
861        assert!(matches!(
862            error,
863            RuntimeError::Storage(StorageError::Timeout { .. })
864        ));
865        assert_eq!(hydrator.admission.available_permits(), 0);
866        assert_eq!(store.calls(), 1, "queued cancellation must start no I/O");
867        drop(first);
868        wait_for_background_task_count(before).await;
869    }
870
871    #[tokio::test(start_paused = true)]
872    #[serial(background_tasks)]
873    async fn caller_deadline_drops_only_the_waiter_until_backend_completion() {
874        let before = crate::background_task_count();
875        let (store, first_started, first_release) =
876            HydrationReadStore::blocking_first(b"x".to_vec());
877        let hydrator = Arc::new(
878            BlobHydrator::new(
879                Arc::clone(&store) as Arc<dyn BlobStore>,
880                khive_storage::MAX_BLOB_WHOLE_BYTES,
881            )
882            .unwrap(),
883        );
884        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
885        let timed_hydrator = Arc::clone(&hydrator);
886        let timed_ref = content_ref.clone();
887        let timed = tokio::spawn(async move {
888            khive_storage::scope_request_read_deadline(
889                std::time::Duration::from_secs(1),
890                timed_hydrator.hydrate_verified(&timed_ref, khive_storage::MAX_BLOB_WHOLE_BYTES),
891            )
892            .await
893        });
894        first_started.await.expect("backend work must start");
895        tokio::time::advance(std::time::Duration::from_secs(1)).await;
896
897        let error = timed.await.unwrap().unwrap_err();
898        assert!(matches!(
899            error,
900            RuntimeError::Storage(StorageError::Timeout { .. })
901        ));
902        assert_eq!(crate::background_task_count(), before + 1);
903
904        let second_hydrator = Arc::clone(&hydrator);
905        let second_ref = content_ref.clone();
906        let second =
907            tokio::spawn(async move { second_hydrator.hydrate_verified(&second_ref, 1).await });
908        assert_eq!(hydrator.admission.available_permits(), 0);
909        assert_eq!(store.calls(), 1);
910
911        first_release.add_permits(1);
912        let verified = second.await.unwrap().unwrap();
913        assert_eq!(verified.bytes(), b"x");
914        drop(verified);
915        wait_for_hydration_calls(&store, 2).await;
916        wait_for_background_task_count(before).await;
917    }
918
919    #[tokio::test]
920    #[serial(background_tasks)]
921    async fn distinct_hydrators_have_independent_admission_budgets() {
922        let before = crate::background_task_count();
923        let (blocked_store, started, release) = HydrationReadStore::blocking_first(b"one".to_vec());
924        let independent_store = HydrationReadStore::immediate(b"two".to_vec());
925        let blocked = Arc::new(
926            BlobHydrator::new(
927                Arc::clone(&blocked_store) as Arc<dyn BlobStore>,
928                khive_storage::MAX_BLOB_WHOLE_BYTES,
929            )
930            .unwrap(),
931        );
932        let independent = BlobHydrator::new(
933            Arc::clone(&independent_store) as Arc<dyn BlobStore>,
934            khive_storage::MAX_BLOB_WHOLE_BYTES,
935        )
936        .unwrap();
937        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
938
939        let blocked_hydrator = Arc::clone(&blocked);
940        let blocked_ref = content_ref.clone();
941        let in_flight = tokio::spawn(async move {
942            blocked_hydrator
943                .hydrate_verified(&blocked_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
944                .await
945        });
946        started.await.expect("first store must enter backend work");
947
948        let verified = independent
949            .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
950            .await
951            .expect("a different store must retain an independent budget");
952        assert_eq!(verified.bytes(), b"two");
953        drop(verified);
954
955        release.add_permits(1);
956        drop(in_flight.await.unwrap().unwrap());
957        wait_for_background_task_count(before).await;
958    }
959
960    #[tokio::test]
961    #[serial(background_tasks)]
962    async fn typed_backend_failure_holds_then_releases_admission() {
963        let before = crate::background_task_count();
964        let (store, started, release) =
965            HydrationReadStore::blocking_first_error(b"recovered".to_vec());
966        let hydrator = Arc::new(
967            BlobHydrator::new(
968                Arc::clone(&store) as Arc<dyn BlobStore>,
969                khive_storage::MAX_BLOB_WHOLE_BYTES,
970            )
971            .unwrap(),
972        );
973        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
974
975        let failing_hydrator = Arc::clone(&hydrator);
976        let failing_ref = content_ref.clone();
977        let failing = tokio::spawn(async move {
978            failing_hydrator
979                .hydrate_verified(&failing_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
980                .await
981        });
982        started.await.expect("failing backend work must start");
983
984        let retry_hydrator = Arc::clone(&hydrator);
985        let retry_ref = content_ref.clone();
986        let retry =
987            tokio::spawn(async move { retry_hydrator.hydrate_verified(&retry_ref, 1).await });
988        assert_eq!(hydrator.admission.available_permits(), 0);
989        assert_eq!(store.calls(), 1);
990
991        release.add_permits(1);
992        let error = failing.await.unwrap().unwrap_err();
993        assert!(matches!(
994            error,
995            RuntimeError::Storage(StorageError::BlobDigestMismatch { .. })
996        ));
997
998        let verified = retry.await.unwrap().unwrap();
999        assert_eq!(verified.bytes(), b"recovered");
1000        assert_eq!(store.calls(), 2);
1001        drop(verified);
1002        wait_for_background_task_count(before).await;
1003    }
1004
1005    #[tokio::test]
1006    #[serial(background_tasks)]
1007    async fn backend_panic_closes_the_reply_and_restores_admission() {
1008        let before = crate::background_task_count();
1009        let (store, started, release) =
1010            HydrationReadStore::blocking_first_panic(b"recovered".to_vec());
1011        let hydrator = Arc::new(
1012            BlobHydrator::new(
1013                Arc::clone(&store) as Arc<dyn BlobStore>,
1014                khive_storage::MAX_BLOB_WHOLE_BYTES,
1015            )
1016            .unwrap(),
1017        );
1018        let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
1019
1020        let panicking_hydrator = Arc::clone(&hydrator);
1021        let panicking_ref = content_ref.clone();
1022        let panicking = tokio::spawn(async move {
1023            panicking_hydrator
1024                .hydrate_verified(&panicking_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
1025                .await
1026        });
1027        started.await.expect("panicking backend work must start");
1028        assert_eq!(crate::background_task_count(), before + 1);
1029        release.add_permits(1);
1030
1031        let error = panicking.await.unwrap().unwrap_err();
1032        assert!(matches!(error, RuntimeError::Internal(_)));
1033        wait_for_background_task_count(before).await;
1034
1035        let recovered = hydrator
1036            .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
1037            .await
1038            .expect("backend panic must not leak admission");
1039        assert_eq!(recovered.bytes(), b"recovered");
1040        assert_eq!(store.calls(), 2);
1041        drop(recovered);
1042        wait_for_background_task_count(before).await;
1043    }
1044
1045    #[test]
1046    fn absent_storage_section_selects_fs_with_explicit_root() {
1047        // An in-memory backend has no data_dir to default beside, so this
1048        // exercises the "existing configurations keep working" path via an
1049        // explicit override rather than proving the full khive#292 chain
1050        // (already covered by `StorageBackend::blob_store`'s own tests).
1051        let dir = tempfile::tempdir().unwrap();
1052        let backend = memory_backend();
1053        let cfg = KhiveConfig::default();
1054        // `resolve_blob_store` with no override falls through to
1055        // `backend.blob_store(None, None)`, which errors for an in-memory
1056        // backend with no root -- confirm that specific, documented failure
1057        // mode rather than silently picking an arbitrary path.
1058        let err = match resolve_blob_store(&cfg, &backend) {
1059            Err(e) => e,
1060            Ok(_) => panic!("expected an error for an in-memory backend with no root override"),
1061        };
1062        assert!(matches!(err, SqliteError::InvalidData(_)));
1063        drop(dir);
1064    }
1065
1066    #[test]
1067    fn explicit_fs_root_is_selected() {
1068        let dir = tempfile::tempdir().unwrap();
1069        let backend = memory_backend();
1070        let cfg = KhiveConfig {
1071            storage: StorageSectionConfig {
1072                blob: Some(BlobConfig::Fs {
1073                    root: Some(dir.path().to_string_lossy().into_owned()),
1074                    floor_bytes: Some(0),
1075                }),
1076            },
1077            ..KhiveConfig::default()
1078        };
1079        let store = resolve_blob_store(&cfg, &backend).expect("fs store should build");
1080        drop(store);
1081    }
1082
1083    #[test]
1084    fn s3_backend_selection_reaches_s3_construction() {
1085        // No AWS credentials in this test process: `S3BlobStore::new` must
1086        // fail at the credential-env check, proving the S3 arm was actually
1087        // selected and reached (not silently falling back to fs).
1088        let _guard = ENV_LOCK.lock().unwrap();
1089        std::env::remove_var("AWS_ACCESS_KEY_ID");
1090        std::env::remove_var("AWS_SECRET_ACCESS_KEY");
1091        let backend = memory_backend();
1092        let cfg = KhiveConfig {
1093            storage: StorageSectionConfig {
1094                blob: Some(BlobConfig::S3 {
1095                    bucket: "khive-blobs".to_string(),
1096                    region: "us-east-1".to_string(),
1097                    endpoint: None,
1098                    prefix: None,
1099                    allow_http: None,
1100                }),
1101            },
1102            ..KhiveConfig::default()
1103        };
1104        let err = match resolve_blob_store(&cfg, &backend) {
1105            Err(e) => e,
1106            Ok(_) => panic!("expected the credential-env error with no AWS env vars set"),
1107        };
1108        let msg = err.to_string();
1109        assert!(
1110            msg.contains("AWS_ACCESS_KEY_ID"),
1111            "expected the credential-env error, got: {msg}"
1112        );
1113    }
1114
1115    // Guards the two credential env vars this module's test toggles, since
1116    // `std::env::set_var`/`remove_var` mutate real process-global state and
1117    // the crate's default parallel test runner would otherwise interleave.
1118    static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
1119}