millipede_core/storage/
mod.rs

1//! Object-safe storage backend contracts.
2
3mod auto_saved;
4mod dataset;
5mod handle;
6mod kvs;
7mod queue;
8pub mod rate_limit;
9
10pub use auto_saved::AutoSaved;
11pub use dataset::{Dataset, DatasetExt, DatasetInfo, ListOptions, Page};
12pub use handle::StorageHandle;
13pub use kvs::{KeyInfo, KeyList, KeyValueStore, KeyValueStoreExt, KvEntry, ListKeysOptions};
14pub use queue::{
15    AddOptions, AddRequestsBatchedResult, BatchAddHandle, Lease, LeaseId, ProcessedRequest,
16    QueueOpInfo, ReclaimOptions, RequestQueue, RequestSource,
17};
18pub use rate_limit::RateLimitReportingClient;
19
20use std::sync::Arc;
21
22/// An error produced by a storage backend.
23#[derive(Debug, thiserror::Error)]
24#[non_exhaustive]
25pub enum StorageError {
26    /// JSON serialization or deserialization failed.
27    #[error("serialization: {0}")]
28    Serialization(#[from] serde_json::Error),
29    /// An input/output operation failed.
30    #[error("io: {0}")]
31    Io(#[from] std::io::Error),
32    /// The referenced lease is no longer active.
33    #[error("lease {lease_id} not found (already completed, abandoned, or expired)")]
34    LeaseNotFound {
35        /// Identifier of the missing lease.
36        lease_id: LeaseId,
37    },
38    /// A backend-specific operation failed.
39    #[error("storage backend: {0}")]
40    Backend(#[source] anyhow::Error),
41    /// The backend does not implement an optional operation.
42    #[error("operation not supported by this backend: {0}")]
43    Unsupported(&'static str),
44    /// The backend rejected an operation because its rate limit was reached.
45    #[error("storage backend rate limited")]
46    RateLimited {
47        /// Suggested delay before retrying, when supplied by the backend.
48        retry_after: Option<std::time::Duration>,
49    },
50}
51
52impl StorageError {
53    /// Returns whether this error represents backend rate limiting.
54    pub fn is_rate_limited(&self) -> bool {
55        matches!(self, StorageError::RateLimited { .. })
56    }
57}
58
59/// Result type returned by every storage operation.
60pub type StorageResult<T> = Result<T, StorageError>;
61
62impl From<StorageError> for crate::errors::CrawlError {
63    fn from(error: StorageError) -> Self {
64        match error {
65            StorageError::Serialization(error) => Self::NonRetryable(anyhow::Error::new(error)),
66            StorageError::Io(error) => Self::Retry(anyhow::Error::new(error)),
67            error @ StorageError::LeaseNotFound { .. } => {
68                Self::NonRetryable(anyhow::Error::new(error))
69            }
70            StorageError::Backend(error) => Self::Retry(error),
71            error @ StorageError::Unsupported(_) => Self::NonRetryable(anyhow::Error::new(error)),
72            error @ StorageError::RateLimited { .. } => Self::Retry(anyhow::Error::new(error)),
73        }
74    }
75}
76
77/// Opens named or default storage objects supplied by a backend.
78#[async_trait::async_trait]
79pub trait StorageClient: Send + Sync + 'static {
80    /// Opens a dataset, using the default dataset when `name` is `None`.
81    async fn open_dataset(&self, name: Option<&str>) -> StorageResult<Arc<dyn Dataset>>;
82    /// Opens a key-value store, using the default store when `name` is `None`.
83    async fn open_key_value_store(
84        &self,
85        name: Option<&str>,
86    ) -> StorageResult<Arc<dyn KeyValueStore>>;
87    /// Opens a request queue, using the default queue when `name` is `None`.
88    async fn open_request_queue(&self, name: Option<&str>) -> StorageResult<Arc<dyn RequestQueue>>;
89    /// Removes all storage data managed by this client.
90    async fn purge(&self) -> StorageResult<()>;
91}
millipede_core/storage/mod.rs

millipede_core/storage/
mod.rs
