1use std::sync::Arc;
10
11use async_trait::async_trait;
12#[cfg(feature = "blob-s3")]
13use khive_db::stores::blob_s3::{S3BlobStore, S3BlobStoreConfig};
14use khive_db::{SqliteError, StorageBackend};
15use khive_storage::{
16 BlobOrphanSweepConfig, BlobOrphanSweepResult, BlobStore, ContentRef, SqlAccess,
17 StorageCapability, StorageError, StorageResult, UploadId,
18};
19
20use crate::engine_config::BlobConfig;
21use crate::{KhiveConfig, RuntimeError, RuntimeResult};
22
23pub const DEFAULT_BLOB_HYDRATION_BYTES: u64 = 4 * khive_storage::MAX_BLOB_WHOLE_BYTES;
25
26#[derive(Debug)]
28pub struct BlobHydrator {
29 store: Arc<dyn BlobStore>,
30 raw_store: Arc<dyn BlobStore>,
34 read_only: bool,
40 governed: bool,
47 admission: Arc<tokio::sync::Semaphore>,
48 budget_bytes: u64,
49}
50
51#[derive(Debug)]
56pub enum GovernedBlobError {
57 Resolve(SqliteError),
59 Construct(RuntimeError),
61}
62
63impl std::fmt::Display for GovernedBlobError {
64 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65 match self {
66 Self::Resolve(error) => write!(f, "blob store resolution: {error}"),
67 Self::Construct(error) => write!(f, "blob hydrator construction: {error}"),
68 }
69 }
70}
71
72impl std::error::Error for GovernedBlobError {}
73
74impl BlobHydrator {
75 pub(crate) fn new_read_only(
80 store: Arc<dyn BlobStore>,
81 budget_bytes: u64,
82 ) -> RuntimeResult<Self> {
83 let mut hydrator =
84 Self::new_inner(wrap_read_only(Arc::clone(&store)), store, budget_bytes)?;
85 hydrator.read_only = true;
86 Ok(hydrator)
87 }
88
89 pub(crate) fn enforces_read_only(&self) -> bool {
91 self.read_only
92 }
93
94 pub(crate) fn raw_store(&self) -> Arc<dyn BlobStore> {
96 Arc::clone(&self.raw_store)
97 }
98
99 pub fn for_mode(
108 store: Arc<dyn BlobStore>,
109 budget_bytes: u64,
110 read_only: bool,
111 ) -> RuntimeResult<Self> {
112 if read_only {
113 Self::new_read_only(store, budget_bytes)
114 } else {
115 Self::new(store, budget_bytes)
116 }
117 }
118
119 pub fn resolve_for_governing_backend(
142 cfg: &KhiveConfig,
143 resolve_backend: &StorageBackend,
144 governing_backend: &StorageBackend,
145 budget_bytes: u64,
146 ) -> Result<Self, GovernedBlobError> {
147 let read_only = governing_backend.is_read_only();
148 let store = resolve_blob_store_for_mode(cfg, resolve_backend, read_only)
149 .map_err(GovernedBlobError::Resolve)?;
150 let mut hydrator =
151 Self::for_mode(store, budget_bytes, read_only).map_err(GovernedBlobError::Construct)?;
152 hydrator.governed = true;
153 Ok(hydrator)
154 }
155
156 pub(crate) fn is_governed(&self) -> bool {
159 self.governed
160 }
161
162 pub fn new(store: Arc<dyn BlobStore>, budget_bytes: u64) -> RuntimeResult<Self> {
164 let raw = Arc::clone(&store);
165 Self::new_inner(store, raw, budget_bytes)
166 }
167
168 fn new_inner(
169 store: Arc<dyn BlobStore>,
170 raw_store: Arc<dyn BlobStore>,
171 budget_bytes: u64,
172 ) -> RuntimeResult<Self> {
173 if budget_bytes < khive_storage::MAX_BLOB_WHOLE_BYTES {
174 return Err(RuntimeError::InvalidInput(format!(
175 "blob hydration budget must be at least {} bytes, got {budget_bytes}",
176 khive_storage::MAX_BLOB_WHOLE_BYTES
177 )));
178 }
179 let permits = usize::try_from(budget_bytes).map_err(|_| {
180 RuntimeError::InvalidInput(format!(
181 "blob hydration budget {budget_bytes} does not fit this platform"
182 ))
183 })?;
184 if permits > tokio::sync::Semaphore::MAX_PERMITS {
185 return Err(RuntimeError::InvalidInput(format!(
186 "blob hydration budget {budget_bytes} exceeds the runtime maximum {}",
187 tokio::sync::Semaphore::MAX_PERMITS
188 )));
189 }
190 Ok(Self {
191 store,
192 raw_store,
193 read_only: false,
194 governed: false,
195 admission: Arc::new(tokio::sync::Semaphore::new(permits)),
196 budget_bytes,
197 })
198 }
199
200 pub fn budget_bytes(&self) -> u64 {
202 self.budget_bytes
203 }
204
205 pub(crate) fn store(&self) -> Arc<dyn BlobStore> {
209 Arc::clone(&self.store)
210 }
211
212 pub async fn hydrate_verified(
214 &self,
215 content_ref: &ContentRef,
216 max_bytes: u64,
217 ) -> RuntimeResult<VerifiedBlob> {
218 if max_bytes > khive_storage::MAX_BLOB_WHOLE_BYTES {
219 return Err(RuntimeError::InvalidInput(format!(
220 "blob hydration max_bytes must not exceed {} bytes, got {max_bytes}",
221 khive_storage::MAX_BLOB_WHOLE_BYTES
222 )));
223 }
224
225 let acquire = Arc::clone(&self.admission).acquire_many_owned(max_bytes as u32);
226 let admission =
227 khive_storage::await_request_read_phase("blob_hydration_admission", acquire)
228 .await?
229 .map_err(|error| {
230 RuntimeError::Internal(format!("blob hydration admission closed: {error}"))
231 })?;
232
233 let (sender, receiver) = tokio::sync::oneshot::channel::<RuntimeResult<VerifiedBlob>>();
234 let store = Arc::clone(&self.store);
235 let content_ref = content_ref.clone();
236 crate::track_named_background_task("blob_hydration", async move {
237 let result = match store.get_bounded_verified(&content_ref, max_bytes).await {
241 Ok(bytes) => Ok(VerifiedBlob {
242 bytes,
243 _admission: admission,
244 }),
245 Err(error) => Err(RuntimeError::Storage(error)),
246 };
247 let _ = sender.send(result);
248 });
249
250 khive_storage::await_request_read_phase("blob_hydration", receiver)
251 .await?
252 .map_err(|_| {
253 RuntimeError::Internal(
254 "blob hydration supervisor ended without delivering a result".to_string(),
255 )
256 })?
257 }
258}
259
260pub struct VerifiedBlob {
289 bytes: Vec<u8>,
290 _admission: tokio::sync::OwnedSemaphorePermit,
291}
292
293impl VerifiedBlob {
294 pub fn bytes(&self) -> &[u8] {
296 &self.bytes
297 }
298}
299
300impl AsRef<[u8]> for VerifiedBlob {
301 fn as_ref(&self) -> &[u8] {
302 self.bytes()
303 }
304}
305
306impl std::fmt::Debug for VerifiedBlob {
307 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
308 formatter
309 .debug_struct("VerifiedBlob")
310 .field("len", &self.bytes.len())
311 .finish_non_exhaustive()
312 }
313}
314
315pub fn resolve_blob_store(
324 cfg: &KhiveConfig,
325 backend: &StorageBackend,
326) -> Result<Arc<dyn BlobStore>, SqliteError> {
327 match &cfg.storage.blob {
328 None => backend.blob_store(None, None),
329 Some(BlobConfig::Fs { root, floor_bytes }) => {
330 let root_path = root.as_ref().map(std::path::PathBuf::from);
331 backend.blob_store(root_path.as_deref(), *floor_bytes)
332 }
333 #[cfg(feature = "blob-s3")]
334 Some(BlobConfig::S3 {
335 bucket,
336 region,
337 endpoint,
338 prefix,
339 allow_http,
340 }) => {
341 let mut s3_cfg = S3BlobStoreConfig::new(bucket.clone(), region.clone());
342 if let Some(endpoint) = endpoint {
343 s3_cfg = s3_cfg.with_endpoint(endpoint.clone());
344 }
345 if let Some(prefix) = prefix {
346 s3_cfg = s3_cfg.with_prefix(prefix.clone());
347 }
348 if let Some(allow_http) = allow_http {
349 s3_cfg = s3_cfg.with_allow_http(*allow_http);
350 }
351 let store = S3BlobStore::new(s3_cfg)?;
352 Ok(Arc::new(store))
353 }
354 #[cfg(not(feature = "blob-s3"))]
355 Some(BlobConfig::S3 { .. }) => Err(SqliteError::InvalidConfig(
356 crate::engine_config::ConfigError::BlobS3FeatureDisabled.to_string(),
357 )),
358 }
359}
360
361pub fn resolve_blob_store_for_mode(
369 cfg: &KhiveConfig,
370 backend: &StorageBackend,
371 read_only: bool,
372) -> Result<Arc<dyn BlobStore>, SqliteError> {
373 if !read_only {
374 return resolve_blob_store(cfg, backend);
375 }
376
377 let inner: Arc<dyn BlobStore> = match &cfg.storage.blob {
378 None => backend.blob_store_read_only(None, None)?,
379 Some(BlobConfig::Fs { root, floor_bytes }) => {
380 let root_path = root.as_ref().map(std::path::PathBuf::from);
381 backend.blob_store_read_only(root_path.as_deref(), *floor_bytes)?
382 }
383 #[cfg(feature = "blob-s3")]
384 Some(BlobConfig::S3 {
385 bucket,
386 region,
387 endpoint,
388 prefix,
389 allow_http,
390 }) => {
391 let mut s3_cfg = S3BlobStoreConfig::new(bucket.clone(), region.clone());
392 if let Some(endpoint) = endpoint {
393 s3_cfg = s3_cfg.with_endpoint(endpoint.clone());
394 }
395 if let Some(prefix) = prefix {
396 s3_cfg = s3_cfg.with_prefix(prefix.clone());
397 }
398 if let Some(allow_http) = allow_http {
399 s3_cfg = s3_cfg.with_allow_http(*allow_http);
400 }
401 Arc::new(S3BlobStore::new(s3_cfg)?)
402 }
403 #[cfg(not(feature = "blob-s3"))]
404 Some(BlobConfig::S3 { .. }) => {
405 return Err(SqliteError::InvalidConfig(
406 crate::engine_config::ConfigError::BlobS3FeatureDisabled.to_string(),
407 ));
408 }
409 };
410 Ok(Arc::new(ReadOnlyBlobStore { inner }))
411}
412
413pub(crate) fn wrap_read_only(inner: Arc<dyn BlobStore>) -> Arc<dyn BlobStore> {
419 Arc::new(ReadOnlyBlobStore { inner })
420}
421
422#[derive(Debug)]
423struct ReadOnlyBlobStore {
424 inner: Arc<dyn BlobStore>,
425}
426
427impl ReadOnlyBlobStore {
428 fn mutation_error(operation: &'static str) -> StorageError {
429 StorageError::Unsupported {
430 capability: StorageCapability::Blob,
431 operation: operation.into(),
432 message: "blob storage is read-only for this pack runtime".to_string(),
433 }
434 }
435}
436
437#[async_trait]
438impl BlobStore for ReadOnlyBlobStore {
439 async fn put(&self, _bytes: Vec<u8>) -> StorageResult<ContentRef> {
440 Err(Self::mutation_error("put"))
441 }
442
443 fn upload_lease_idle_cap(&self) -> Option<std::time::Duration> {
444 self.inner.upload_lease_idle_cap()
445 }
446
447 async fn begin_upload_with_lease(
448 &self,
449 _size: u64,
450 _config: khive_storage::blob::UploadLeaseConfig,
451 ) -> StorageResult<UploadId> {
452 Err(Self::mutation_error("begin_upload_with_lease"))
453 }
454
455 async fn renew_upload(&self, _id: &UploadId) -> StorageResult<()> {
456 Err(Self::mutation_error("renew_upload"))
457 }
458
459 async fn begin_upload(&self, _declared_size: u64) -> StorageResult<UploadId> {
460 Err(Self::mutation_error("begin_upload"))
461 }
462
463 async fn append_part(&self, _id: &UploadId, _bytes: Vec<u8>) -> StorageResult<u64> {
464 Err(Self::mutation_error("append_part"))
465 }
466
467 async fn commit_upload(&self, _id: &UploadId, _content_ref: &ContentRef) -> StorageResult<()> {
468 Err(Self::mutation_error("commit_upload"))
469 }
470
471 async fn abort_upload(&self, _id: &UploadId) -> StorageResult<()> {
472 Err(Self::mutation_error("abort_upload"))
473 }
474
475 async fn sweep_uploads(&self, _idle_for: std::time::Duration) -> StorageResult<u64> {
476 Err(Self::mutation_error("sweep_uploads"))
477 }
478
479 async fn get_bounded_verified(
480 &self,
481 content_ref: &ContentRef,
482 max_bytes: u64,
483 ) -> StorageResult<Vec<u8>> {
484 self.inner
485 .get_bounded_verified(content_ref, max_bytes)
486 .await
487 }
488
489 async fn exists(&self, content_ref: &ContentRef) -> StorageResult<bool> {
490 self.inner.exists(content_ref).await
491 }
492
493 async fn size(&self, content_ref: &ContentRef) -> StorageResult<Option<u64>> {
494 self.inner.size(content_ref).await
495 }
496
497 async fn delete(&self, _content_ref: &ContentRef) -> StorageResult<bool> {
498 Err(Self::mutation_error("delete"))
499 }
500
501 async fn orphan_sweep(
502 &self,
503 _config: &BlobOrphanSweepConfig,
504 ) -> StorageResult<BlobOrphanSweepResult> {
505 Err(Self::mutation_error("orphan_sweep"))
506 }
507
508 async fn transactional_orphan_sweep(
509 &self,
510 _sql: &dyn SqlAccess,
511 _dry_run: bool,
512 ) -> StorageResult<BlobOrphanSweepResult> {
513 Err(Self::mutation_error("transactional_orphan_sweep"))
514 }
515}
516
517#[cfg(test)]
518#[path = "blob_tests.rs"]
519mod tests;