1use std::sync::Arc;
10
11use async_trait::async_trait;
12use khive_db::stores::blob_s3::{S3BlobStore, S3BlobStoreConfig};
13use khive_db::{SqliteError, StorageBackend};
14use khive_storage::{
15 BlobOrphanSweepConfig, BlobOrphanSweepResult, BlobStore, ContentRef, SqlAccess,
16 StorageCapability, StorageError, StorageResult,
17};
18
19use crate::engine_config::BlobConfig;
20use crate::{KhiveConfig, RuntimeError, RuntimeResult};
21
22pub const DEFAULT_BLOB_HYDRATION_BYTES: u64 = 4 * khive_storage::MAX_BLOB_WHOLE_BYTES;
24
25#[derive(Debug)]
27pub struct BlobHydrator {
28 store: Arc<dyn BlobStore>,
29 raw_store: Arc<dyn BlobStore>,
33 read_only: bool,
39 governed: bool,
46 admission: Arc<tokio::sync::Semaphore>,
47 budget_bytes: u64,
48}
49
50#[derive(Debug)]
55pub enum GovernedBlobError {
56 Resolve(SqliteError),
58 Construct(RuntimeError),
60}
61
62impl std::fmt::Display for GovernedBlobError {
63 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
64 match self {
65 Self::Resolve(error) => write!(f, "blob store resolution: {error}"),
66 Self::Construct(error) => write!(f, "blob hydrator construction: {error}"),
67 }
68 }
69}
70
71impl std::error::Error for GovernedBlobError {}
72
73impl BlobHydrator {
74 pub(crate) fn new_read_only(
79 store: Arc<dyn BlobStore>,
80 budget_bytes: u64,
81 ) -> RuntimeResult<Self> {
82 let mut hydrator =
83 Self::new_inner(wrap_read_only(Arc::clone(&store)), store, budget_bytes)?;
84 hydrator.read_only = true;
85 Ok(hydrator)
86 }
87
88 pub(crate) fn enforces_read_only(&self) -> bool {
90 self.read_only
91 }
92
93 pub(crate) fn raw_store(&self) -> Arc<dyn BlobStore> {
95 Arc::clone(&self.raw_store)
96 }
97
98 pub fn for_mode(
107 store: Arc<dyn BlobStore>,
108 budget_bytes: u64,
109 read_only: bool,
110 ) -> RuntimeResult<Self> {
111 if read_only {
112 Self::new_read_only(store, budget_bytes)
113 } else {
114 Self::new(store, budget_bytes)
115 }
116 }
117
118 pub fn resolve_for_governing_backend(
141 cfg: &KhiveConfig,
142 resolve_backend: &StorageBackend,
143 governing_backend: &StorageBackend,
144 budget_bytes: u64,
145 ) -> Result<Self, GovernedBlobError> {
146 let read_only = governing_backend.is_read_only();
147 let store = resolve_blob_store_for_mode(cfg, resolve_backend, read_only)
148 .map_err(GovernedBlobError::Resolve)?;
149 let mut hydrator =
150 Self::for_mode(store, budget_bytes, read_only).map_err(GovernedBlobError::Construct)?;
151 hydrator.governed = true;
152 Ok(hydrator)
153 }
154
155 pub(crate) fn is_governed(&self) -> bool {
158 self.governed
159 }
160
161 pub fn new(store: Arc<dyn BlobStore>, budget_bytes: u64) -> RuntimeResult<Self> {
163 let raw = Arc::clone(&store);
164 Self::new_inner(store, raw, budget_bytes)
165 }
166
167 fn new_inner(
168 store: Arc<dyn BlobStore>,
169 raw_store: Arc<dyn BlobStore>,
170 budget_bytes: u64,
171 ) -> RuntimeResult<Self> {
172 if budget_bytes < khive_storage::MAX_BLOB_WHOLE_BYTES {
173 return Err(RuntimeError::InvalidInput(format!(
174 "blob hydration budget must be at least {} bytes, got {budget_bytes}",
175 khive_storage::MAX_BLOB_WHOLE_BYTES
176 )));
177 }
178 let permits = usize::try_from(budget_bytes).map_err(|_| {
179 RuntimeError::InvalidInput(format!(
180 "blob hydration budget {budget_bytes} does not fit this platform"
181 ))
182 })?;
183 if permits > tokio::sync::Semaphore::MAX_PERMITS {
184 return Err(RuntimeError::InvalidInput(format!(
185 "blob hydration budget {budget_bytes} exceeds the runtime maximum {}",
186 tokio::sync::Semaphore::MAX_PERMITS
187 )));
188 }
189 Ok(Self {
190 store,
191 raw_store,
192 read_only: false,
193 governed: false,
194 admission: Arc::new(tokio::sync::Semaphore::new(permits)),
195 budget_bytes,
196 })
197 }
198
199 pub fn budget_bytes(&self) -> u64 {
201 self.budget_bytes
202 }
203
204 pub(crate) fn store(&self) -> Arc<dyn BlobStore> {
208 Arc::clone(&self.store)
209 }
210
211 pub async fn hydrate_verified(
213 &self,
214 content_ref: &ContentRef,
215 max_bytes: u64,
216 ) -> RuntimeResult<VerifiedBlob> {
217 if max_bytes > khive_storage::MAX_BLOB_WHOLE_BYTES {
218 return Err(RuntimeError::InvalidInput(format!(
219 "blob hydration max_bytes must not exceed {} bytes, got {max_bytes}",
220 khive_storage::MAX_BLOB_WHOLE_BYTES
221 )));
222 }
223
224 let acquire = Arc::clone(&self.admission).acquire_many_owned(max_bytes as u32);
225 let admission =
226 khive_storage::await_request_read_phase("blob_hydration_admission", acquire)
227 .await?
228 .map_err(|error| {
229 RuntimeError::Internal(format!("blob hydration admission closed: {error}"))
230 })?;
231
232 let (sender, receiver) = tokio::sync::oneshot::channel::<RuntimeResult<VerifiedBlob>>();
233 let store = Arc::clone(&self.store);
234 let content_ref = content_ref.clone();
235 crate::track_background_task(async move {
236 let result = match store.get_bounded_verified(&content_ref, max_bytes).await {
240 Ok(bytes) => Ok(VerifiedBlob {
241 bytes,
242 _admission: admission,
243 }),
244 Err(error) => Err(RuntimeError::Storage(error)),
245 };
246 let _ = sender.send(result);
247 });
248
249 khive_storage::await_request_read_phase("blob_hydration", receiver)
250 .await?
251 .map_err(|_| {
252 RuntimeError::Internal(
253 "blob hydration supervisor ended without delivering a result".to_string(),
254 )
255 })?
256 }
257}
258
259pub struct VerifiedBlob {
288 bytes: Vec<u8>,
289 _admission: tokio::sync::OwnedSemaphorePermit,
290}
291
292impl VerifiedBlob {
293 pub fn bytes(&self) -> &[u8] {
295 &self.bytes
296 }
297}
298
299impl AsRef<[u8]> for VerifiedBlob {
300 fn as_ref(&self) -> &[u8] {
301 self.bytes()
302 }
303}
304
305impl std::fmt::Debug for VerifiedBlob {
306 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
307 formatter
308 .debug_struct("VerifiedBlob")
309 .field("len", &self.bytes.len())
310 .finish_non_exhaustive()
311 }
312}
313
314pub fn resolve_blob_store(
323 cfg: &KhiveConfig,
324 backend: &StorageBackend,
325) -> Result<Arc<dyn BlobStore>, SqliteError> {
326 match &cfg.storage.blob {
327 None => backend.blob_store(None, None),
328 Some(BlobConfig::Fs { root, floor_bytes }) => {
329 let root_path = root.as_ref().map(std::path::PathBuf::from);
330 backend.blob_store(root_path.as_deref(), *floor_bytes)
331 }
332 Some(BlobConfig::S3 {
333 bucket,
334 region,
335 endpoint,
336 prefix,
337 allow_http,
338 }) => {
339 let mut s3_cfg = S3BlobStoreConfig::new(bucket.clone(), region.clone());
340 if let Some(endpoint) = endpoint {
341 s3_cfg = s3_cfg.with_endpoint(endpoint.clone());
342 }
343 if let Some(prefix) = prefix {
344 s3_cfg = s3_cfg.with_prefix(prefix.clone());
345 }
346 if let Some(allow_http) = allow_http {
347 s3_cfg = s3_cfg.with_allow_http(*allow_http);
348 }
349 let store = S3BlobStore::new(s3_cfg)?;
350 Ok(Arc::new(store))
351 }
352 }
353}
354
355pub fn resolve_blob_store_for_mode(
363 cfg: &KhiveConfig,
364 backend: &StorageBackend,
365 read_only: bool,
366) -> Result<Arc<dyn BlobStore>, SqliteError> {
367 if !read_only {
368 return resolve_blob_store(cfg, backend);
369 }
370
371 let inner: Arc<dyn BlobStore> = match &cfg.storage.blob {
372 None => backend.blob_store_read_only(None, None)?,
373 Some(BlobConfig::Fs { root, floor_bytes }) => {
374 let root_path = root.as_ref().map(std::path::PathBuf::from);
375 backend.blob_store_read_only(root_path.as_deref(), *floor_bytes)?
376 }
377 Some(BlobConfig::S3 {
378 bucket,
379 region,
380 endpoint,
381 prefix,
382 allow_http,
383 }) => {
384 let mut s3_cfg = S3BlobStoreConfig::new(bucket.clone(), region.clone());
385 if let Some(endpoint) = endpoint {
386 s3_cfg = s3_cfg.with_endpoint(endpoint.clone());
387 }
388 if let Some(prefix) = prefix {
389 s3_cfg = s3_cfg.with_prefix(prefix.clone());
390 }
391 if let Some(allow_http) = allow_http {
392 s3_cfg = s3_cfg.with_allow_http(*allow_http);
393 }
394 Arc::new(S3BlobStore::new(s3_cfg)?)
395 }
396 };
397 Ok(Arc::new(ReadOnlyBlobStore { inner }))
398}
399
400pub(crate) fn wrap_read_only(inner: Arc<dyn BlobStore>) -> Arc<dyn BlobStore> {
406 Arc::new(ReadOnlyBlobStore { inner })
407}
408
409#[derive(Debug)]
410struct ReadOnlyBlobStore {
411 inner: Arc<dyn BlobStore>,
412}
413
414impl ReadOnlyBlobStore {
415 fn mutation_error(operation: &'static str) -> StorageError {
416 StorageError::Unsupported {
417 capability: StorageCapability::Blob,
418 operation: operation.into(),
419 message: "blob storage is read-only for this pack runtime".to_string(),
420 }
421 }
422}
423
424#[async_trait]
425impl BlobStore for ReadOnlyBlobStore {
426 async fn put(&self, _bytes: Vec<u8>) -> StorageResult<ContentRef> {
427 Err(Self::mutation_error("put"))
428 }
429
430 async fn get_bounded_verified(
431 &self,
432 content_ref: &ContentRef,
433 max_bytes: u64,
434 ) -> StorageResult<Vec<u8>> {
435 self.inner
436 .get_bounded_verified(content_ref, max_bytes)
437 .await
438 }
439
440 async fn exists(&self, content_ref: &ContentRef) -> StorageResult<bool> {
441 self.inner.exists(content_ref).await
442 }
443
444 async fn size(&self, content_ref: &ContentRef) -> StorageResult<Option<u64>> {
445 self.inner.size(content_ref).await
446 }
447
448 async fn delete(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
449 Err(Self::mutation_error("delete"))
450 }
451
452 async fn orphan_sweep(
453 &self,
454 _config: &BlobOrphanSweepConfig,
455 ) -> StorageResult<BlobOrphanSweepResult> {
456 Err(Self::mutation_error("orphan_sweep"))
457 }
458
459 async fn transactional_orphan_sweep(
460 &self,
461 _sql: &dyn SqlAccess,
462 _dry_run: bool,
463 ) -> StorageResult<BlobOrphanSweepResult> {
464 Err(Self::mutation_error("transactional_orphan_sweep"))
465 }
466}
467
468#[cfg(test)]
469mod tests {
470 use super::*;
471 use crate::engine_config::StorageSectionConfig;
472 use serial_test::serial;
473
474 #[derive(Debug, Default)]
475 struct RecordingReadStore {
476 bounded_call: std::sync::Mutex<Option<(ContentRef, u64)>>,
477 }
478
479 #[async_trait]
480 impl BlobStore for RecordingReadStore {
481 async fn put(&self, _bytes: Vec<u8>) -> StorageResult<ContentRef> {
482 panic!("put is not used by the read-only delegation test")
483 }
484
485 async fn get_bounded_verified(
486 &self,
487 content_ref: &ContentRef,
488 max_bytes: u64,
489 ) -> StorageResult<Vec<u8>> {
490 *self.bounded_call.lock().unwrap() = Some((content_ref.clone(), max_bytes));
491 Ok(b"verified".to_vec())
492 }
493
494 async fn exists(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
495 Ok(true)
496 }
497
498 async fn size(&self, _content_ref: &ContentRef) -> StorageResult<Option<u64>> {
499 Ok(Some(8))
500 }
501
502 async fn delete(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
503 panic!("delete is not used by the read-only delegation test")
504 }
505 }
506
507 fn memory_backend() -> StorageBackend {
508 StorageBackend::memory().expect("memory backend should create")
509 }
510
511 #[tokio::test]
512 async fn read_only_store_delegates_the_bounded_verified_read_unchanged() {
513 let inner = Arc::new(RecordingReadStore::default());
514 let store = ReadOnlyBlobStore {
515 inner: Arc::clone(&inner) as Arc<dyn BlobStore>,
516 };
517 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
518
519 let bytes = store.get_bounded_verified(&content_ref, 17).await.unwrap();
520 assert_eq!(bytes, b"verified");
521 assert_eq!(*inner.bounded_call.lock().unwrap(), Some((content_ref, 17)));
522 }
523
524 #[derive(Debug)]
525 struct HydrationReadStore {
526 calls: std::sync::atomic::AtomicUsize,
527 bytes: Vec<u8>,
528 first_started: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
529 first_release: Option<Arc<tokio::sync::Semaphore>>,
530 fail_first: bool,
531 panic_first: bool,
532 }
533
534 impl HydrationReadStore {
535 fn immediate(bytes: Vec<u8>) -> Arc<Self> {
536 Arc::new(Self {
537 calls: std::sync::atomic::AtomicUsize::new(0),
538 bytes,
539 first_started: std::sync::Mutex::new(None),
540 first_release: None,
541 fail_first: false,
542 panic_first: false,
543 })
544 }
545
546 fn blocking_first(
547 bytes: Vec<u8>,
548 ) -> (
549 Arc<Self>,
550 tokio::sync::oneshot::Receiver<()>,
551 Arc<tokio::sync::Semaphore>,
552 ) {
553 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
554 let release = Arc::new(tokio::sync::Semaphore::new(0));
555 (
556 Arc::new(Self {
557 calls: std::sync::atomic::AtomicUsize::new(0),
558 bytes,
559 first_started: std::sync::Mutex::new(Some(started_tx)),
560 first_release: Some(Arc::clone(&release)),
561 fail_first: false,
562 panic_first: false,
563 }),
564 started_rx,
565 release,
566 )
567 }
568
569 fn blocking_first_panic(
570 bytes: Vec<u8>,
571 ) -> (
572 Arc<Self>,
573 tokio::sync::oneshot::Receiver<()>,
574 Arc<tokio::sync::Semaphore>,
575 ) {
576 let (store, started, release) = Self::blocking_first(bytes);
577 let store = Arc::try_unwrap(store).expect("new fixture has one owner");
578 (
579 Arc::new(Self {
580 panic_first: true,
581 ..store
582 }),
583 started,
584 release,
585 )
586 }
587
588 fn blocking_first_error(
589 bytes: Vec<u8>,
590 ) -> (
591 Arc<Self>,
592 tokio::sync::oneshot::Receiver<()>,
593 Arc<tokio::sync::Semaphore>,
594 ) {
595 let (store, started, release) = Self::blocking_first(bytes);
596 let store = Arc::try_unwrap(store).expect("new fixture has one owner");
597 (
598 Arc::new(Self {
599 fail_first: true,
600 ..store
601 }),
602 started,
603 release,
604 )
605 }
606
607 fn calls(&self) -> usize {
608 self.calls.load(std::sync::atomic::Ordering::SeqCst)
609 }
610 }
611
612 #[async_trait]
613 impl BlobStore for HydrationReadStore {
614 async fn put(&self, _bytes: Vec<u8>) -> StorageResult<ContentRef> {
615 panic!("put is not used by hydrator tests")
616 }
617
618 async fn get_bounded_verified(
619 &self,
620 content_ref: &ContentRef,
621 _max_bytes: u64,
622 ) -> StorageResult<Vec<u8>> {
623 let call = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
624 if call == 0 {
625 if let Some(started) = self.first_started.lock().unwrap().take() {
626 let _ = started.send(());
627 }
628 if let Some(release) = &self.first_release {
629 release
630 .clone()
631 .acquire_owned()
632 .await
633 .expect("test release semaphore stays open")
634 .forget();
635 }
636 if self.fail_first {
637 return Err(StorageError::BlobDigestMismatch {
638 expected: content_ref.clone(),
639 actual: ContentRef::from_hex("b".repeat(64))
640 .expect("fixture digest is canonical"),
641 });
642 }
643 assert!(!self.panic_first, "injected hydration backend panic");
644 }
645 Ok(self.bytes.clone())
646 }
647
648 async fn exists(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
649 panic!("exists is not used by hydrator tests")
650 }
651
652 async fn size(&self, _content_ref: &ContentRef) -> StorageResult<Option<u64>> {
653 panic!("size is not used by hydrator tests")
654 }
655
656 async fn delete(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
657 panic!("delete is not used by hydrator tests")
658 }
659 }
660
661 async fn wait_for_hydration_calls(store: &HydrationReadStore, expected: usize) {
662 tokio::time::timeout(std::time::Duration::from_secs(1), async {
663 while store.calls() != expected {
664 tokio::task::yield_now().await;
665 }
666 })
667 .await
668 .expect("hydration backend call count should advance");
669 }
670
671 async fn wait_for_background_task_count(expected: usize) {
672 for _ in 0..10_000 {
673 if crate::background_task_count() == expected {
674 return;
675 }
676 tokio::task::yield_now().await;
677 }
678 assert_eq!(crate::background_task_count(), expected);
679 }
680
681 #[tokio::test]
682 async fn hydrator_rejects_an_invalid_maximum_before_admission_or_backend_work() {
683 let store = HydrationReadStore::immediate(b"x".to_vec());
684 let hydrator = BlobHydrator::new(
685 Arc::clone(&store) as Arc<dyn BlobStore>,
686 khive_storage::MAX_BLOB_WHOLE_BYTES,
687 )
688 .unwrap();
689 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
690
691 let error = hydrator
692 .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES + 1)
693 .await
694 .unwrap_err();
695
696 assert!(matches!(error, crate::RuntimeError::InvalidInput(_)));
697 assert_eq!(store.calls(), 0);
698 }
699
700 #[tokio::test]
701 #[serial(background_tasks)]
702 async fn zero_maximum_and_idle_portable_maximum_are_both_admissible() {
703 let before = crate::background_task_count();
704 let empty_store = HydrationReadStore::immediate(Vec::new());
705 let empty_hydrator = BlobHydrator::new(
706 Arc::clone(&empty_store) as Arc<dyn BlobStore>,
707 khive_storage::MAX_BLOB_WHOLE_BYTES,
708 )
709 .unwrap();
710 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
711
712 let empty = empty_hydrator
713 .hydrate_verified(&content_ref, 0)
714 .await
715 .expect("zero is a valid declared maximum");
716 assert!(empty.bytes().is_empty());
717 drop(empty);
718
719 let max_store = HydrationReadStore::immediate(b"bounded".to_vec());
720 let max_hydrator = BlobHydrator::new(
721 Arc::clone(&max_store) as Arc<dyn BlobStore>,
722 khive_storage::MAX_BLOB_WHOLE_BYTES,
723 )
724 .unwrap();
725 let max = max_hydrator
726 .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
727 .await
728 .expect("an idle minimum-size budget admits every valid maximum");
729 assert_eq!(max.bytes(), b"bounded");
730 drop(max);
731 wait_for_background_task_count(before).await;
732 }
733
734 #[tokio::test]
735 #[serial(background_tasks)]
736 async fn cancelling_a_waiter_retains_admission_until_backend_completion() {
737 let before = crate::background_task_count();
738 let (store, first_started, first_release) =
739 HydrationReadStore::blocking_first(b"x".to_vec());
740 let hydrator = Arc::new(
741 BlobHydrator::new(
742 Arc::clone(&store) as Arc<dyn BlobStore>,
743 khive_storage::MAX_BLOB_WHOLE_BYTES,
744 )
745 .unwrap(),
746 );
747 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
748
749 let first_hydrator = Arc::clone(&hydrator);
750 let first_ref = content_ref.clone();
751 let first = tokio::spawn(async move {
752 first_hydrator
753 .hydrate_verified(&first_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
754 .await
755 });
756 first_started.await.expect("first backend read must start");
757 assert_eq!(crate::background_task_count(), before + 1);
758
759 first.abort();
760 assert!(first.await.unwrap_err().is_cancelled());
761
762 let second_hydrator = Arc::clone(&hydrator);
763 let second_ref = content_ref.clone();
764 let second =
765 tokio::spawn(async move { second_hydrator.hydrate_verified(&second_ref, 1).await });
766 assert_eq!(hydrator.admission.available_permits(), 0);
767 assert_eq!(
768 store.calls(),
769 1,
770 "cancelled waiters must not release admission held by native work"
771 );
772
773 first_release.add_permits(1);
774 let verified = tokio::time::timeout(std::time::Duration::from_secs(1), second)
775 .await
776 .expect("second waiter should acquire after native completion")
777 .expect("second waiter task should not panic")
778 .expect("second hydration should succeed");
779 assert_eq!(verified.bytes(), b"x");
780 drop(verified);
781 wait_for_hydration_calls(&store, 2).await;
782
783 wait_for_background_task_count(before).await;
784 }
785
786 #[tokio::test]
787 #[serial(background_tasks)]
788 async fn verified_blob_retains_its_weighted_lease_until_drop() {
789 let before = crate::background_task_count();
790 let store = HydrationReadStore::immediate(b"x".to_vec());
791 let hydrator = Arc::new(
792 BlobHydrator::new(
793 Arc::clone(&store) as Arc<dyn BlobStore>,
794 khive_storage::MAX_BLOB_WHOLE_BYTES,
795 )
796 .unwrap(),
797 );
798 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
799
800 let first = hydrator
801 .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
802 .await
803 .unwrap();
804 assert_eq!(first.bytes(), b"x");
805
806 let second_hydrator = Arc::clone(&hydrator);
807 let second_ref = content_ref.clone();
808 let second =
809 tokio::spawn(async move { second_hydrator.hydrate_verified(&second_ref, 1).await });
810 assert_eq!(hydrator.admission.available_permits(), 0);
811 assert_eq!(
812 store.calls(),
813 1,
814 "borrowed access must not release the original raw-buffer lease"
815 );
816
817 drop(first);
818 let second = tokio::time::timeout(std::time::Duration::from_secs(1), second)
819 .await
820 .expect("dropping VerifiedBlob should release its lease")
821 .expect("second waiter task should not panic")
822 .expect("second hydration should succeed");
823 assert_eq!(second.bytes(), b"x");
824 drop(second);
825 wait_for_background_task_count(before).await;
826 }
827
828 #[tokio::test]
829 #[serial(background_tasks)]
830 async fn cancelling_while_queued_starts_no_backend_work() {
831 let before = crate::background_task_count();
832 let store = HydrationReadStore::immediate(b"x".to_vec());
833 let hydrator = Arc::new(
834 BlobHydrator::new(
835 Arc::clone(&store) as Arc<dyn BlobStore>,
836 khive_storage::MAX_BLOB_WHOLE_BYTES,
837 )
838 .unwrap(),
839 );
840 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
841 let first = hydrator
842 .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
843 .await
844 .unwrap();
845
846 let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false);
847 let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
848 let queued_hydrator = Arc::clone(&hydrator);
849 let queued_ref = content_ref.clone();
850 let queued = tokio::spawn(async move {
851 khive_storage::scope_request_read_cancellation(cancel_rx, async move {
852 let _ = entered_tx.send(());
853 queued_hydrator.hydrate_verified(&queued_ref, 1).await
854 })
855 .await
856 });
857 entered_rx.await.expect("queued request must be polled");
858 cancel_tx.send(true).unwrap();
859
860 let error = queued.await.unwrap().unwrap_err();
861 assert!(matches!(
862 error,
863 RuntimeError::Storage(StorageError::Timeout { .. })
864 ));
865 assert_eq!(hydrator.admission.available_permits(), 0);
866 assert_eq!(store.calls(), 1, "queued cancellation must start no I/O");
867 drop(first);
868 wait_for_background_task_count(before).await;
869 }
870
871 #[tokio::test(start_paused = true)]
872 #[serial(background_tasks)]
873 async fn caller_deadline_drops_only_the_waiter_until_backend_completion() {
874 let before = crate::background_task_count();
875 let (store, first_started, first_release) =
876 HydrationReadStore::blocking_first(b"x".to_vec());
877 let hydrator = Arc::new(
878 BlobHydrator::new(
879 Arc::clone(&store) as Arc<dyn BlobStore>,
880 khive_storage::MAX_BLOB_WHOLE_BYTES,
881 )
882 .unwrap(),
883 );
884 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
885 let timed_hydrator = Arc::clone(&hydrator);
886 let timed_ref = content_ref.clone();
887 let timed = tokio::spawn(async move {
888 khive_storage::scope_request_read_deadline(
889 std::time::Duration::from_secs(1),
890 timed_hydrator.hydrate_verified(&timed_ref, khive_storage::MAX_BLOB_WHOLE_BYTES),
891 )
892 .await
893 });
894 first_started.await.expect("backend work must start");
895 tokio::time::advance(std::time::Duration::from_secs(1)).await;
896
897 let error = timed.await.unwrap().unwrap_err();
898 assert!(matches!(
899 error,
900 RuntimeError::Storage(StorageError::Timeout { .. })
901 ));
902 assert_eq!(crate::background_task_count(), before + 1);
903
904 let second_hydrator = Arc::clone(&hydrator);
905 let second_ref = content_ref.clone();
906 let second =
907 tokio::spawn(async move { second_hydrator.hydrate_verified(&second_ref, 1).await });
908 assert_eq!(hydrator.admission.available_permits(), 0);
909 assert_eq!(store.calls(), 1);
910
911 first_release.add_permits(1);
912 let verified = second.await.unwrap().unwrap();
913 assert_eq!(verified.bytes(), b"x");
914 drop(verified);
915 wait_for_hydration_calls(&store, 2).await;
916 wait_for_background_task_count(before).await;
917 }
918
919 #[tokio::test]
920 #[serial(background_tasks)]
921 async fn distinct_hydrators_have_independent_admission_budgets() {
922 let before = crate::background_task_count();
923 let (blocked_store, started, release) = HydrationReadStore::blocking_first(b"one".to_vec());
924 let independent_store = HydrationReadStore::immediate(b"two".to_vec());
925 let blocked = Arc::new(
926 BlobHydrator::new(
927 Arc::clone(&blocked_store) as Arc<dyn BlobStore>,
928 khive_storage::MAX_BLOB_WHOLE_BYTES,
929 )
930 .unwrap(),
931 );
932 let independent = BlobHydrator::new(
933 Arc::clone(&independent_store) as Arc<dyn BlobStore>,
934 khive_storage::MAX_BLOB_WHOLE_BYTES,
935 )
936 .unwrap();
937 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
938
939 let blocked_hydrator = Arc::clone(&blocked);
940 let blocked_ref = content_ref.clone();
941 let in_flight = tokio::spawn(async move {
942 blocked_hydrator
943 .hydrate_verified(&blocked_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
944 .await
945 });
946 started.await.expect("first store must enter backend work");
947
948 let verified = independent
949 .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
950 .await
951 .expect("a different store must retain an independent budget");
952 assert_eq!(verified.bytes(), b"two");
953 drop(verified);
954
955 release.add_permits(1);
956 drop(in_flight.await.unwrap().unwrap());
957 wait_for_background_task_count(before).await;
958 }
959
960 #[tokio::test]
961 #[serial(background_tasks)]
962 async fn typed_backend_failure_holds_then_releases_admission() {
963 let before = crate::background_task_count();
964 let (store, started, release) =
965 HydrationReadStore::blocking_first_error(b"recovered".to_vec());
966 let hydrator = Arc::new(
967 BlobHydrator::new(
968 Arc::clone(&store) as Arc<dyn BlobStore>,
969 khive_storage::MAX_BLOB_WHOLE_BYTES,
970 )
971 .unwrap(),
972 );
973 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
974
975 let failing_hydrator = Arc::clone(&hydrator);
976 let failing_ref = content_ref.clone();
977 let failing = tokio::spawn(async move {
978 failing_hydrator
979 .hydrate_verified(&failing_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
980 .await
981 });
982 started.await.expect("failing backend work must start");
983
984 let retry_hydrator = Arc::clone(&hydrator);
985 let retry_ref = content_ref.clone();
986 let retry =
987 tokio::spawn(async move { retry_hydrator.hydrate_verified(&retry_ref, 1).await });
988 assert_eq!(hydrator.admission.available_permits(), 0);
989 assert_eq!(store.calls(), 1);
990
991 release.add_permits(1);
992 let error = failing.await.unwrap().unwrap_err();
993 assert!(matches!(
994 error,
995 RuntimeError::Storage(StorageError::BlobDigestMismatch { .. })
996 ));
997
998 let verified = retry.await.unwrap().unwrap();
999 assert_eq!(verified.bytes(), b"recovered");
1000 assert_eq!(store.calls(), 2);
1001 drop(verified);
1002 wait_for_background_task_count(before).await;
1003 }
1004
1005 #[tokio::test]
1006 #[serial(background_tasks)]
1007 async fn backend_panic_closes_the_reply_and_restores_admission() {
1008 let before = crate::background_task_count();
1009 let (store, started, release) =
1010 HydrationReadStore::blocking_first_panic(b"recovered".to_vec());
1011 let hydrator = Arc::new(
1012 BlobHydrator::new(
1013 Arc::clone(&store) as Arc<dyn BlobStore>,
1014 khive_storage::MAX_BLOB_WHOLE_BYTES,
1015 )
1016 .unwrap(),
1017 );
1018 let content_ref = ContentRef::from_hex("a".repeat(64)).unwrap();
1019
1020 let panicking_hydrator = Arc::clone(&hydrator);
1021 let panicking_ref = content_ref.clone();
1022 let panicking = tokio::spawn(async move {
1023 panicking_hydrator
1024 .hydrate_verified(&panicking_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
1025 .await
1026 });
1027 started.await.expect("panicking backend work must start");
1028 assert_eq!(crate::background_task_count(), before + 1);
1029 release.add_permits(1);
1030
1031 let error = panicking.await.unwrap().unwrap_err();
1032 assert!(matches!(error, RuntimeError::Internal(_)));
1033 wait_for_background_task_count(before).await;
1034
1035 let recovered = hydrator
1036 .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
1037 .await
1038 .expect("backend panic must not leak admission");
1039 assert_eq!(recovered.bytes(), b"recovered");
1040 assert_eq!(store.calls(), 2);
1041 drop(recovered);
1042 wait_for_background_task_count(before).await;
1043 }
1044
1045 #[test]
1046 fn absent_storage_section_selects_fs_with_explicit_root() {
1047 let dir = tempfile::tempdir().unwrap();
1052 let backend = memory_backend();
1053 let cfg = KhiveConfig::default();
1054 let err = match resolve_blob_store(&cfg, &backend) {
1059 Err(e) => e,
1060 Ok(_) => panic!("expected an error for an in-memory backend with no root override"),
1061 };
1062 assert!(matches!(err, SqliteError::InvalidData(_)));
1063 drop(dir);
1064 }
1065
1066 #[test]
1067 fn explicit_fs_root_is_selected() {
1068 let dir = tempfile::tempdir().unwrap();
1069 let backend = memory_backend();
1070 let cfg = KhiveConfig {
1071 storage: StorageSectionConfig {
1072 blob: Some(BlobConfig::Fs {
1073 root: Some(dir.path().to_string_lossy().into_owned()),
1074 floor_bytes: Some(0),
1075 }),
1076 },
1077 ..KhiveConfig::default()
1078 };
1079 let store = resolve_blob_store(&cfg, &backend).expect("fs store should build");
1080 drop(store);
1081 }
1082
1083 #[test]
1084 fn s3_backend_selection_reaches_s3_construction() {
1085 let _guard = ENV_LOCK.lock().unwrap();
1089 std::env::remove_var("AWS_ACCESS_KEY_ID");
1090 std::env::remove_var("AWS_SECRET_ACCESS_KEY");
1091 let backend = memory_backend();
1092 let cfg = KhiveConfig {
1093 storage: StorageSectionConfig {
1094 blob: Some(BlobConfig::S3 {
1095 bucket: "khive-blobs".to_string(),
1096 region: "us-east-1".to_string(),
1097 endpoint: None,
1098 prefix: None,
1099 allow_http: None,
1100 }),
1101 },
1102 ..KhiveConfig::default()
1103 };
1104 let err = match resolve_blob_store(&cfg, &backend) {
1105 Err(e) => e,
1106 Ok(_) => panic!("expected the credential-env error with no AWS env vars set"),
1107 };
1108 let msg = err.to_string();
1109 assert!(
1110 msg.contains("AWS_ACCESS_KEY_ID"),
1111 "expected the credential-env error, got: {msg}"
1112 );
1113 }
1114
1115 static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
1119}