1use std::{
4 collections::BTreeMap,
5 future::Future,
6 sync::{
7 Arc,
8 atomic::{AtomicU64, Ordering},
9 },
10 time::{Duration, Instant},
11};
12
13#[cfg(feature = "local")]
14use std::collections::BTreeSet;
15#[cfg(feature = "local")]
16use std::path::PathBuf;
17
18use async_trait::async_trait;
19use futures::StreamExt as _;
20use futures::future::BoxFuture;
21use imbl::OrdMap;
22use tokio::sync::Mutex;
23
24use crate::{limits, wire};
25
26const MEMORY_BYTES: usize = 64 * 1_024 * 1_024;
27const SYSTEM_METADATA_BYTES: usize = 2 * 1_024;
28const MAX_LISTING_VIEWS: usize = 1_024;
29const MAX_CONCURRENT_BATCH_READS: usize = 16;
30static PROVIDER_SEQUENCE: AtomicU64 = AtomicU64::new(1);
31
32async fn indexed<F: Future>(index: usize, future: F) -> (usize, F::Output) {
33 (index, future.await)
34}
35
36async fn collect_ordered_bounded<I, F, T>(items: I, concurrency: usize) -> Vec<T>
37where
38 I: IntoIterator<Item = F>,
39 F: Future<Output = T>,
40{
41 let mut pending = items.into_iter().enumerate();
42 let mut active = futures::stream::FuturesUnordered::new();
43 for _ in 0..concurrency {
44 let Some((index, future)) = pending.next() else {
45 break;
46 };
47 active.push(indexed(index, future));
48 }
49 let mut results = Vec::new();
50 while let Some(result) = active.next().await {
51 results.push(result);
52 if let Some((index, future)) = pending.next() {
53 active.push(indexed(index, future));
54 }
55 }
56 results.sort_unstable_by_key(|(index, _)| *index);
57 results.into_iter().map(|(_, result)| result).collect()
58}
59
60#[derive(Clone, Debug, Eq, PartialEq)]
62pub enum Condition {
63 IfAbsent,
65 IfMatch(String),
67 IfVersion(String),
69}
70
71#[cfg(any(feature = "grpc", feature = "local"))]
72impl Condition {
73 pub(crate) fn wire(self) -> wire::Preconditions {
74 let condition = match self {
75 Self::IfAbsent => wire::preconditions::Condition::IfAbsent(true),
76 Self::IfMatch(etag) => wire::preconditions::Condition::IfMatch(etag),
77 Self::IfVersion(version_id) => wire::preconditions::Condition::IfVersion(version_id),
78 };
79 wire::Preconditions {
80 condition: Some(condition),
81 }
82 }
83}
84
85#[derive(Clone, Debug, PartialEq)]
87pub enum ReadTarget {
88 Bucket(wire::BucketRef),
90 Snapshot(wire::SnapshotRef),
92}
93
94#[derive(Clone, Debug)]
96pub struct PutRequest {
97 pub bucket: wire::BucketRef,
99 pub object_key: String,
101 pub body: bytes::Bytes,
103 pub metadata: wire::ObjectMetadata,
105 pub condition: Option<Condition>,
107 pub idempotency_key: Option<String>,
109}
110
111#[derive(Clone, Debug)]
113pub struct GetRequest {
114 pub target: ReadTarget,
116 pub object_key: String,
118 pub version_id: Option<String>,
120 pub range: Option<(u64, Option<u64>)>,
122 pub if_match: Option<String>,
124 pub if_none_match: Option<String>,
126 pub maximum_bytes: u64,
128}
129
130#[derive(Clone, Debug, PartialEq)]
132pub struct BufferedObject {
133 pub version: wire::ObjectVersion,
135 pub body: bytes::Bytes,
137}
138
139#[derive(Clone, Debug, PartialEq)]
141pub struct DeleteResult {
142 pub existed: bool,
144 pub marker: Option<wire::ObjectVersion>,
146}
147
148#[derive(Clone, Debug, PartialEq)]
150pub struct ProviderListPage {
151 pub entries: Vec<wire::ListEntry>,
153 pub common_prefixes: Vec<String>,
155 pub continuation: Option<String>,
157}
158
159#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)]
161pub enum ObjectsError {
162 #[error("invalid objects request: {0}")]
164 Invalid(&'static str),
165 #[error("objects resource not found")]
167 NotFound,
168 #[error("objects resource already exists")]
170 AlreadyExists,
171 #[error("objects precondition failed")]
173 PreconditionFailed,
174 #[error("objects idempotency mismatch")]
176 IdempotencyMismatch,
177 #[error("objects capacity exhausted")]
179 Capacity,
180 #[error("objects capability is unsupported")]
182 Unsupported,
183 #[error("objects provider rejected authorization")]
185 Unauthorized,
186 #[error("objects provider is unavailable")]
188 Unavailable,
189}
190
191#[async_trait]
193pub trait ObjectsProvider: Send + Sync {
194 async fn create_bucket(
196 &self,
197 name: String,
198 idempotency_key: Option<String>,
199 ) -> Result<wire::Bucket, ObjectsError>;
200 async fn head_bucket(&self, bucket: &wire::BucketRef) -> Result<wire::Bucket, ObjectsError>;
202 async fn delete_bucket(
204 &self,
205 bucket: &wire::BucketRef,
206 idempotency_key: Option<String>,
207 ) -> Result<bool, ObjectsError>;
208 async fn put(&self, request: PutRequest) -> Result<wire::ObjectVersion, ObjectsError>;
210 async fn put_batch(
216 &self,
217 requests: Vec<PutRequest>,
218 ) -> Vec<Result<wire::ObjectVersion, ObjectsError>> {
219 let mut results = Vec::with_capacity(requests.len());
220 for request in requests {
221 results.push(self.put(request).await);
222 }
223 results
224 }
225 async fn get(&self, request: GetRequest) -> Result<BufferedObject, ObjectsError>;
227 async fn get_batch(
233 &self,
234 requests: Vec<GetRequest>,
235 ) -> Vec<Result<BufferedObject, ObjectsError>> {
236 let mut results = Vec::with_capacity(requests.len());
237 for request in requests {
238 results.push(self.get(request).await);
239 }
240 results
241 }
242 async fn delete(
244 &self,
245 bucket: wire::BucketRef,
246 object_key: String,
247 version_id: Option<String>,
248 condition: Option<Condition>,
249 idempotency_key: Option<String>,
250 ) -> Result<DeleteResult, ObjectsError>;
251 async fn list(
253 &self,
254 target: ReadTarget,
255 prefix: String,
256 delimiter: Option<String>,
257 versions: bool,
258 page_size: u32,
259 continuation: Option<String>,
260 ) -> Result<ProviderListPage, ObjectsError>;
261 async fn snapshot(
263 &self,
264 bucket: wire::BucketRef,
265 idempotency_key: Option<String>,
266 ) -> Result<wire::Snapshot, ObjectsError>;
267 async fn destroy_snapshot(
269 &self,
270 snapshot: wire::SnapshotRef,
271 idempotency_key: Option<String>,
272 ) -> Result<bool, ObjectsError>;
273 async fn fork(
275 &self,
276 source: ReadTarget,
277 destination_name: String,
278 idempotency_key: Option<String>,
279 ) -> Result<wire::Bucket, ObjectsError>;
280 async fn create_multipart(
282 &self,
283 bucket: wire::BucketRef,
284 object_key: String,
285 metadata: wire::ObjectMetadata,
286 condition: Option<Condition>,
287 idempotency_key: Option<String>,
288 ) -> Result<wire::MultipartUpload, ObjectsError>;
289 async fn upload_part(
291 &self,
292 bucket: wire::BucketRef,
293 object_key: String,
294 upload_id: String,
295 part_number: u32,
296 body: bytes::Bytes,
297 idempotency_key: Option<String>,
298 ) -> Result<wire::UploadedPart, ObjectsError>;
299 async fn list_parts(
301 &self,
302 bucket: wire::BucketRef,
303 object_key: String,
304 upload_id: String,
305 ) -> Result<Vec<wire::UploadedPart>, ObjectsError>;
306 async fn complete_multipart(
308 &self,
309 bucket: wire::BucketRef,
310 object_key: String,
311 upload_id: String,
312 parts: Vec<wire::UploadedPart>,
313 idempotency_key: Option<String>,
314 ) -> Result<wire::ObjectVersion, ObjectsError>;
315 async fn abort_multipart(
317 &self,
318 bucket: wire::BucketRef,
319 object_key: String,
320 upload_id: String,
321 idempotency_key: Option<String>,
322 ) -> Result<bool, ObjectsError>;
323}
324
325#[derive(Clone)]
326enum StoredBody {
327 Memory(bytes::Bytes),
328 Composite {
329 parts: Arc<[StoredBody]>,
330 length: usize,
331 },
332 #[cfg(feature = "local")]
333 Local {
334 root: Arc<PathBuf>,
335 digest: [u8; 32],
336 length: usize,
337 location: LocalBodyLocation,
338 },
339}
340
341#[derive(Clone, Eq, Ord, PartialEq, PartialOrd)]
342enum BodyIdentity {
343 Memory {
344 address: usize,
345 length: usize,
346 },
347 #[cfg(feature = "local")]
348 Local([u8; 32]),
349}
350
351impl StoredBody {
352 fn memory(body: bytes::Bytes) -> Self {
353 Self::Memory(body)
354 }
355
356 fn len(&self) -> usize {
357 match self {
358 Self::Memory(body) => body.len(),
359 Self::Composite { length, .. } => *length,
360 #[cfg(feature = "local")]
361 Self::Local { length, .. } => *length,
362 }
363 }
364
365 fn update_hash(&self, hasher: &mut blake3::Hasher) -> Result<(), ObjectsError> {
366 match self {
367 Self::Memory(body) => {
368 hasher.update(body);
369 Ok(())
370 }
371 Self::Composite { parts, .. } => {
372 for part in parts.iter() {
373 part.update_hash(hasher)?;
374 }
375 Ok(())
376 }
377 #[cfg(feature = "local")]
378 Self::Local {
379 root,
380 digest,
381 length,
382 location,
383 } => crate::local::hash_body_at(root, digest, *length, location, hasher),
384 }
385 }
386
387 fn leaves(&self, output: &mut Vec<(BodyIdentity, usize)>) {
388 match self {
389 Self::Memory(body) => output.push((
390 BodyIdentity::Memory {
391 address: body.as_ptr() as usize,
392 length: body.len(),
393 },
394 body.len(),
395 )),
396 Self::Composite { parts, .. } => {
397 for part in parts.iter() {
398 part.leaves(output);
399 }
400 }
401 #[cfg(feature = "local")]
402 Self::Local { digest, length, .. } => {
403 output.push((BodyIdentity::Local(*digest), *length));
404 }
405 }
406 }
407
408 #[cfg(feature = "local")]
409 fn local_references(&self, output: &mut BTreeSet<LocalBodyReference>) {
410 match self {
411 Self::Memory(_) => {}
412 Self::Composite { parts, .. } => {
413 for part in parts.iter() {
414 part.local_references(output);
415 }
416 }
417 Self::Local {
418 digest,
419 length,
420 location,
421 ..
422 } => {
423 output.insert(LocalBodyReference {
424 digest: *digest,
425 length: *length,
426 location: location.clone(),
427 });
428 }
429 }
430 }
431
432 fn read_async(
433 &self,
434 start: usize,
435 end: usize,
436 ) -> BoxFuture<'_, Result<bytes::Bytes, ObjectsError>> {
437 Box::pin(async move {
438 match self {
439 Self::Memory(body) => Ok(body.slice(start..end)),
440 Self::Composite { parts, .. } => {
441 let mut output = Vec::with_capacity(end.saturating_sub(start));
442 let mut offset = 0usize;
443 for part in parts.iter() {
444 let part_end = offset
445 .checked_add(part.len())
446 .ok_or(ObjectsError::Unavailable)?;
447 if part_end > start && offset < end {
448 let selected_start = start.saturating_sub(offset).min(part.len());
449 let selected_end = end.saturating_sub(offset).min(part.len());
450 output.extend_from_slice(
451 &part.read_async(selected_start, selected_end).await?,
452 );
453 }
454 offset = part_end;
455 }
456 if output.len() != end.saturating_sub(start) {
457 return Err(ObjectsError::Unavailable);
458 }
459 Ok(output.into())
460 }
461 #[cfg(feature = "local")]
462 Self::Local {
463 root,
464 digest,
465 length,
466 location,
467 } => {
468 crate::local::read_body_at_async(root, digest, *length, location, start, end)
469 .await
470 }
471 }
472 })
473 }
474}
475
476#[derive(Clone)]
477struct Version {
478 descriptor: wire::ObjectVersion,
479 body: Option<StoredBody>,
480}
481
482struct ResolvedGet {
483 descriptor: wire::ObjectVersion,
484 body: StoredBody,
485 start: usize,
486 end: usize,
487}
488
489#[derive(Clone)]
490struct BucketState {
491 reference: wire::BucketRef,
492 created_at: prost_types::Timestamp,
493 objects: OrdMap<String, Vec<Version>>,
494 listing: OrdMap<String, Vec<wire::ObjectVersion>>,
495}
496
497impl BucketState {
498 fn append(&mut self, key: String, version: Version) {
499 self.listing
500 .entry(key.clone())
501 .or_default()
502 .push(version.descriptor.clone());
503 self.objects.entry(key).or_default().push(version);
504 }
505
506 fn remove_exact(&mut self, key: &str, identity: &str) -> Result<Option<Version>, ObjectsError> {
507 let Some(history) = self.objects.get_mut(key) else {
508 return Ok(None);
509 };
510 let Some(position) = history
511 .iter()
512 .position(|version| version.descriptor.version_id == identity)
513 else {
514 return Ok(None);
515 };
516 let removed = history.remove(position);
517 let descriptors = self.listing.get_mut(key).ok_or(ObjectsError::Unavailable)?;
518 if descriptors.get(position) != Some(&removed.descriptor) {
519 return Err(ObjectsError::Unavailable);
520 }
521 descriptors.remove(position);
522 if history.is_empty() {
523 self.objects.remove(key);
524 self.listing.remove(key);
525 }
526 Ok(Some(removed))
527 }
528}
529
530#[derive(Clone)]
531struct SnapshotState {
532 reference: wire::SnapshotRef,
533 bucket: BucketState,
534}
535
536struct MultipartState {
537 bucket: wire::BucketRef,
538 object_key: String,
539 metadata: wire::ObjectMetadata,
540 condition: Option<Condition>,
541 parts: BTreeMap<u32, (wire::UploadedPart, StoredBody)>,
542}
543
544struct StoredPartRequest {
545 bucket: wire::BucketRef,
546 object_key: String,
547 upload_id: String,
548 part_number: u32,
549 body: StoredBody,
550 body_digest: [u8; 32],
551 idempotency_key: Option<String>,
552}
553
554#[cfg(feature = "local")]
555pub(crate) struct ExternalBody {
556 pub(crate) root: PathBuf,
557 pub(crate) digest: [u8; 32],
558 pub(crate) length: usize,
559 pub(crate) location: LocalBodyLocation,
560}
561
562#[cfg(feature = "local")]
563#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
564pub(crate) enum LocalBodyLocation {
565 Segment { id: [u8; 32], offset: u64 },
566}
567
568#[cfg(feature = "local")]
569#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
570pub(crate) struct LocalBodyReference {
571 pub(crate) digest: [u8; 32],
572 pub(crate) length: usize,
573 pub(crate) location: LocalBodyLocation,
574}
575
576enum ListingItem {
577 Entry(Box<wire::ListEntry>),
578 Prefix(String),
579}
580
581struct ListingView {
582 binding: ListingBinding,
583 objects: OrdMap<String, Vec<wire::ObjectVersion>>,
584 expires_at: Instant,
585 prefetched: Option<PrefetchedListingItem>,
586}
587
588struct PrefetchedListingItem {
589 offset: usize,
590 cursor: ListingCursor,
591 item: ListingItem,
592 after: ListingCursor,
593}
594
595#[derive(Clone, PartialEq)]
596struct ListingBinding {
597 target: ReadTarget,
598 prefix: String,
599 delimiter: Option<String>,
600 versions: bool,
601}
602
603#[derive(Clone, Eq, PartialEq)]
604struct ListingCursor {
605 key: Option<String>,
606 version_index: usize,
607}
608
609#[derive(Clone)]
610enum MutationOutcome {
611 Bucket(Result<wire::Bucket, ObjectsError>),
612 Boolean(Result<bool, ObjectsError>),
613 Version(Result<wire::ObjectVersion, ObjectsError>),
614 Delete(Result<DeleteResult, ObjectsError>),
615 Snapshot(Result<wire::Snapshot, ObjectsError>),
616 Multipart(Result<wire::MultipartUpload, ObjectsError>),
617 Part(Result<wire::UploadedPart, ObjectsError>),
618}
619
620struct IdempotencyRecord {
621 fingerprint: blake3::Hash,
622 outcome: MutationOutcome,
623}
624
625struct State {
626 maximum_bytes: usize,
627 maximum_object_bytes: usize,
628 sequence: u64,
629 listing_sequence: u64,
630 listing_epoch: u64,
631 bytes: usize,
632 body_references: BTreeMap<BodyIdentity, (usize, usize)>,
633 names: BTreeMap<String, String>,
634 buckets: BTreeMap<String, BucketState>,
635 snapshots: BTreeMap<String, SnapshotState>,
636 multiparts: BTreeMap<String, MultipartState>,
637 listings: BTreeMap<String, ListingView>,
638 idempotency: BTreeMap<String, IdempotencyRecord>,
639}
640
641impl Default for State {
642 fn default() -> Self {
643 Self {
644 maximum_bytes: MEMORY_BYTES,
645 maximum_object_bytes: MEMORY_BYTES,
646 sequence: 0,
647 listing_sequence: 0,
648 listing_epoch: PROVIDER_SEQUENCE.fetch_add(1, Ordering::Relaxed),
649 bytes: 0,
650 body_references: BTreeMap::new(),
651 names: BTreeMap::new(),
652 buckets: BTreeMap::new(),
653 snapshots: BTreeMap::new(),
654 multiparts: BTreeMap::new(),
655 listings: BTreeMap::new(),
656 idempotency: BTreeMap::new(),
657 }
658 }
659}
660
661impl State {
662 fn additional_body_bytes(&self, body: &StoredBody) -> Result<usize, ObjectsError> {
663 let mut leaves = Vec::new();
664 body.leaves(&mut leaves);
665 leaves
666 .into_iter()
667 .try_fold(0usize, |total, (identity, length)| {
668 if self.body_references.contains_key(&identity) {
669 Ok(total)
670 } else {
671 total.checked_add(length).ok_or(ObjectsError::Capacity)
672 }
673 })
674 }
675
676 fn retain_body(&mut self, body: &StoredBody) -> Result<(), ObjectsError> {
677 let mut leaves = Vec::new();
678 body.leaves(&mut leaves);
679 for (identity, length) in leaves {
680 match self.body_references.entry(identity) {
681 std::collections::btree_map::Entry::Vacant(entry) => {
682 self.bytes = self
683 .bytes
684 .checked_add(length)
685 .ok_or(ObjectsError::Capacity)?;
686 entry.insert((length, 1));
687 }
688 std::collections::btree_map::Entry::Occupied(mut entry) => {
689 if entry.get().0 != length {
690 return Err(ObjectsError::Unavailable);
691 }
692 entry.get_mut().1 =
693 entry.get().1.checked_add(1).ok_or(ObjectsError::Capacity)?;
694 }
695 }
696 }
697 Ok(())
698 }
699
700 fn release_body(&mut self, body: &StoredBody) -> Result<(), ObjectsError> {
701 let mut leaves = Vec::new();
702 body.leaves(&mut leaves);
703 for (identity, length) in leaves {
704 let references = self
705 .body_references
706 .get_mut(&identity)
707 .ok_or(ObjectsError::Unavailable)?;
708 if references.0 != length || references.1 == 0 {
709 return Err(ObjectsError::Unavailable);
710 }
711 references.1 -= 1;
712 if references.1 == 0 {
713 self.body_references.remove(&identity);
714 self.bytes = self
715 .bytes
716 .checked_sub(length)
717 .ok_or(ObjectsError::Unavailable)?;
718 }
719 }
720 Ok(())
721 }
722
723 fn retain_bucket(&mut self, bucket: &BucketState) -> Result<(), ObjectsError> {
724 for body in bucket
725 .objects
726 .values()
727 .flatten()
728 .filter_map(|version| version.body.as_ref())
729 {
730 self.retain_body(body)?;
731 }
732 Ok(())
733 }
734
735 fn release_bucket(&mut self, bucket: &BucketState) -> Result<(), ObjectsError> {
736 for body in bucket
737 .objects
738 .values()
739 .flatten()
740 .filter_map(|version| version.body.as_ref())
741 {
742 self.release_body(body)?;
743 }
744 Ok(())
745 }
746}
747
748#[derive(Clone, Default)]
753pub struct MemoryObjects {
754 state: Arc<Mutex<State>>,
755}
756
757impl MemoryObjects {
758 #[cfg(feature = "local")]
759 pub(crate) async fn local_body_references(&self) -> BTreeSet<LocalBodyReference> {
760 let state = self.state.lock().await;
761 let mut references = BTreeSet::new();
762 for bucket in state
763 .buckets
764 .values()
765 .chain(state.snapshots.values().map(|snapshot| &snapshot.bucket))
766 {
767 for body in bucket
768 .objects
769 .values()
770 .flatten()
771 .filter_map(|version| version.body.as_ref())
772 {
773 body.local_references(&mut references);
774 }
775 }
776 for body in state
777 .multiparts
778 .values()
779 .flat_map(|upload| upload.parts.values().map(|(_, body)| body))
780 {
781 body.local_references(&mut references);
782 }
783 references
784 }
785
786 pub async fn bucket_named(&self, name: &str) -> Result<Option<wire::BucketRef>, ObjectsError> {
788 Self::validate_bucket_name(name)?;
789 let state = self.state.lock().await;
790 let Some(bucket_id) = state.names.get(name) else {
791 return Ok(None);
792 };
793 let bucket = state
794 .buckets
795 .get(bucket_id)
796 .ok_or(ObjectsError::Unavailable)?;
797 Ok(Some(bucket.reference.clone()))
798 }
799
800 #[must_use]
802 pub fn with_default_bucket() -> (Self, wire::BucketRef) {
803 Self::from_bucket("memory-bucket".to_owned(), MEMORY_BYTES)
804 }
805
806 pub fn new(maximum_bytes: u64) -> Result<Self, ObjectsError> {
812 if maximum_bytes == 0 {
813 return Err(ObjectsError::Invalid("memory byte limit must be positive"));
814 }
815 Self::new_with_limits(maximum_bytes, maximum_bytes)
816 }
817
818 pub(crate) fn new_with_limits(
819 maximum_object_bytes: u64,
820 maximum_bytes: u64,
821 ) -> Result<Self, ObjectsError> {
822 let maximum_object_bytes = usize::try_from(maximum_object_bytes)
823 .map_err(|_| ObjectsError::Invalid("memory object limit is not representable"))?;
824 let maximum_bytes = usize::try_from(maximum_bytes)
825 .map_err(|_| ObjectsError::Invalid("memory byte limit is not representable"))?;
826 if maximum_object_bytes == 0 || maximum_bytes == 0 || maximum_object_bytes > maximum_bytes {
827 return Err(ObjectsError::Invalid("invalid memory object limits"));
828 }
829 Ok(Self {
830 state: Arc::new(Mutex::new(State {
831 maximum_bytes,
832 maximum_object_bytes,
833 ..State::default()
834 })),
835 })
836 }
837
838 pub fn with_bucket(
847 name: impl Into<String>,
848 maximum_bytes: u64,
849 ) -> Result<(Self, wire::BucketRef), ObjectsError> {
850 let maximum_bytes = usize::try_from(maximum_bytes)
851 .map_err(|_| ObjectsError::Invalid("memory byte limit is not representable"))?;
852 if maximum_bytes == 0 {
853 return Err(ObjectsError::Invalid("memory byte limit must be positive"));
854 }
855 let name = name.into();
856 Self::validate_bucket_name(&name)?;
857 Ok(Self::from_bucket(name, maximum_bytes))
858 }
859
860 pub fn with_bucket_limits(
867 name: impl Into<String>,
868 maximum_object_bytes: u64,
869 maximum_bytes: u64,
870 ) -> Result<(Self, wire::BucketRef), ObjectsError> {
871 let maximum_object_bytes = usize::try_from(maximum_object_bytes)
872 .map_err(|_| ObjectsError::Invalid("memory object limit is not representable"))?;
873 let maximum_bytes = usize::try_from(maximum_bytes)
874 .map_err(|_| ObjectsError::Invalid("memory byte limit is not representable"))?;
875 if maximum_object_bytes == 0 || maximum_bytes == 0 || maximum_object_bytes > maximum_bytes {
876 return Err(ObjectsError::Invalid("invalid memory object limits"));
877 }
878 let name = name.into();
879 Self::validate_bucket_name(&name)?;
880 Ok(Self::from_bucket_limits(
881 name,
882 maximum_object_bytes,
883 maximum_bytes,
884 ))
885 }
886
887 fn from_bucket(name: String, maximum_bytes: usize) -> (Self, wire::BucketRef) {
888 Self::from_bucket_limits(name, maximum_bytes, maximum_bytes)
889 }
890
891 fn from_bucket_limits(
892 name: String,
893 maximum_object_bytes: usize,
894 maximum_bytes: usize,
895 ) -> (Self, wire::BucketRef) {
896 let bucket_id = "bucket-0000000000000001".to_owned();
897 let reference = wire::BucketRef {
898 bucket_id: bucket_id.clone(),
899 name: name.clone(),
900 };
901 let mut state = State {
902 maximum_bytes,
903 maximum_object_bytes,
904 sequence: 1,
905 ..State::default()
906 };
907 state.names.insert(name, bucket_id.clone());
908 state.buckets.insert(
909 bucket_id,
910 BucketState {
911 reference: reference.clone(),
912 created_at: Self::timestamp(&state),
913 objects: OrdMap::new(),
914 listing: OrdMap::new(),
915 },
916 );
917 (
918 Self {
919 state: Arc::new(Mutex::new(state)),
920 },
921 reference,
922 )
923 }
924
925 fn fingerprint(kind: &str, fields: &[&[u8]]) -> blake3::Hash {
926 let mut hasher = blake3::Hasher::new();
927 for field in std::iter::once(kind.as_bytes()).chain(fields.iter().copied()) {
928 hasher.update(&(field.len() as u64).to_le_bytes());
929 hasher.update(field);
930 }
931 hasher.finalize()
932 }
933
934 fn execute<T: Clone>(
935 state: &mut State,
936 idempotency_key: Option<&str>,
937 fingerprint: blake3::Hash,
938 unpack: impl FnOnce(&MutationOutcome) -> Option<Result<T, ObjectsError>>,
939 pack: impl FnOnce(Result<T, ObjectsError>) -> MutationOutcome,
940 operation: impl FnOnce(&mut State) -> Result<T, ObjectsError>,
941 ) -> Result<T, ObjectsError> {
942 let Some(idempotency_key) = idempotency_key else {
943 return operation(state);
944 };
945 if idempotency_key.is_empty() || idempotency_key.len() > 256 {
946 return Err(ObjectsError::Invalid("invalid idempotency key"));
947 }
948 if let Some(record) = state.idempotency.get(idempotency_key) {
949 if record.fingerprint != fingerprint {
950 return Err(ObjectsError::IdempotencyMismatch);
951 }
952 return unpack(&record.outcome).ok_or(ObjectsError::IdempotencyMismatch)?;
953 }
954 let outcome = operation(state);
955 state.idempotency.insert(
956 idempotency_key.to_owned(),
957 IdempotencyRecord {
958 fingerprint,
959 outcome: pack(outcome.clone()),
960 },
961 );
962 outcome
963 }
964
965 fn reference_fields(reference: &wire::BucketRef) -> [&[u8]; 2] {
966 [reference.bucket_id.as_bytes(), reference.name.as_bytes()]
967 }
968
969 fn condition_bytes(condition: &Option<Condition>) -> Vec<u8> {
970 match condition {
971 None => b"none".to_vec(),
972 Some(Condition::IfAbsent) => b"absent".to_vec(),
973 Some(Condition::IfMatch(value)) => [b"etag:".as_slice(), value.as_bytes()].concat(),
974 Some(Condition::IfVersion(value)) => {
975 [b"version:".as_slice(), value.as_bytes()].concat()
976 }
977 }
978 }
979
980 fn metadata_bytes(metadata: &wire::ObjectMetadata) -> Vec<u8> {
981 let mut fields = vec![
982 metadata.content_type.as_str(),
983 metadata.content_encoding.as_str(),
984 metadata.cache_control.as_str(),
985 metadata.content_disposition.as_str(),
986 metadata.content_language.as_str(),
987 ];
988 fields.push(if metadata.expires_unix_seconds.is_some() {
989 "expires"
990 } else {
991 "no-expires"
992 });
993 let mut encoded = Vec::new();
994 for field in fields {
995 encoded.extend_from_slice(&(field.len() as u64).to_le_bytes());
996 encoded.extend_from_slice(field.as_bytes());
997 }
998 if let Some(expires) = metadata.expires_unix_seconds {
999 encoded.extend_from_slice(&expires.to_le_bytes());
1000 }
1001 let mut user = metadata.user.iter().collect::<Vec<_>>();
1002 user.sort_unstable_by(|left, right| left.0.cmp(right.0));
1003 for (name, value) in user {
1004 for field in [name.as_bytes(), value.as_bytes()] {
1005 encoded.extend_from_slice(&(field.len() as u64).to_le_bytes());
1006 encoded.extend_from_slice(field);
1007 }
1008 }
1009 encoded
1010 }
1011
1012 fn next(state: &mut State, kind: &str) -> Result<String, ObjectsError> {
1013 let sequence = if kind == "listing" {
1014 &mut state.listing_sequence
1015 } else {
1016 &mut state.sequence
1017 };
1018 *sequence = sequence.checked_add(1).ok_or(ObjectsError::Capacity)?;
1019 Ok(format!("{kind}-{sequence:016x}"))
1020 }
1021
1022 fn timestamp(state: &State) -> prost_types::Timestamp {
1023 prost_types::Timestamp {
1024 seconds: i64::try_from(state.sequence).unwrap_or(i64::MAX),
1025 nanos: 0,
1026 }
1027 }
1028
1029 fn bucket<'a>(
1030 state: &'a State,
1031 reference: &wire::BucketRef,
1032 ) -> Result<&'a BucketState, ObjectsError> {
1033 state
1034 .buckets
1035 .get(&reference.bucket_id)
1036 .filter(|bucket| bucket.reference == *reference)
1037 .ok_or(ObjectsError::NotFound)
1038 }
1039
1040 fn bucket_mut<'a>(
1041 state: &'a mut State,
1042 reference: &wire::BucketRef,
1043 ) -> Result<&'a mut BucketState, ObjectsError> {
1044 state
1045 .buckets
1046 .get_mut(&reference.bucket_id)
1047 .filter(|bucket| bucket.reference == *reference)
1048 .ok_or(ObjectsError::NotFound)
1049 }
1050
1051 fn validate_key(value: &str) -> Result<(), ObjectsError> {
1052 if value.is_empty() || value.len() > limits::KEY_BYTES || value.contains('\0') {
1053 Err(ObjectsError::Invalid("invalid object key"))
1054 } else {
1055 Ok(())
1056 }
1057 }
1058
1059 fn validate_bucket_name(name: &str) -> Result<(), ObjectsError> {
1060 let valid_edges = name
1061 .as_bytes()
1062 .first()
1063 .zip(name.as_bytes().last())
1064 .is_some_and(|(first, last)| {
1065 first.is_ascii_alphanumeric() && last.is_ascii_alphanumeric()
1066 });
1067 let valid_body = name
1068 .bytes()
1069 .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-');
1070 if !(3..=63).contains(&name.len()) || !valid_edges || !valid_body {
1071 Err(ObjectsError::Invalid("invalid bucket name"))
1072 } else {
1073 Ok(())
1074 }
1075 }
1076
1077 fn validate_metadata(metadata: &wire::ObjectMetadata) -> Result<(), ObjectsError> {
1078 let fixed = [
1079 &metadata.content_type,
1080 &metadata.content_encoding,
1081 &metadata.cache_control,
1082 &metadata.content_disposition,
1083 &metadata.content_language,
1084 ]
1085 .into_iter()
1086 .map(String::len)
1087 .sum::<usize>();
1088 let user = metadata
1089 .user
1090 .iter()
1091 .map(|(name, value)| name.len() + value.len())
1092 .sum::<usize>();
1093 if fixed > SYSTEM_METADATA_BYTES
1094 || user > limits::USER_METADATA_BYTES
1095 || metadata.user.iter().any(|(name, value)| {
1096 !name.bytes().all(|byte| {
1097 byte.is_ascii_alphanumeric()
1098 || matches!(
1099 byte,
1100 b'!' | b'#'
1101 | b'$'
1102 | b'%'
1103 | b'&'
1104 | b'\''
1105 | b'*'
1106 | b'+'
1107 | b'-'
1108 | b'.'
1109 | b'^'
1110 | b'_'
1111 | b'`'
1112 | b'|'
1113 | b'~'
1114 )
1115 }) || name.is_empty()
1116 || !Self::valid_header_value(value)
1117 })
1118 || [
1119 &metadata.content_type,
1120 &metadata.content_encoding,
1121 &metadata.cache_control,
1122 &metadata.content_disposition,
1123 &metadata.content_language,
1124 ]
1125 .into_iter()
1126 .any(|value| !Self::valid_header_value(value))
1127 || metadata
1128 .expires_unix_seconds
1129 .is_some_and(|seconds| !(-62_167_219_200..=253_402_300_799).contains(&seconds))
1130 {
1131 Err(ObjectsError::Invalid("invalid object metadata"))
1132 } else {
1133 Ok(())
1134 }
1135 }
1136
1137 fn valid_header_value(value: &str) -> bool {
1138 value
1139 .bytes()
1140 .all(|byte| byte == b'\t' || (0x20..=0x7e).contains(&byte))
1141 }
1142
1143 fn target(state: &State, target: &ReadTarget) -> Result<BucketState, ObjectsError> {
1144 match target {
1145 ReadTarget::Bucket(reference) => Ok(Self::bucket(state, reference)?.clone()),
1146 ReadTarget::Snapshot(reference) => state
1147 .snapshots
1148 .get(&reference.snapshot_id)
1149 .filter(|snapshot| snapshot.reference == *reference)
1150 .map(|snapshot| snapshot.bucket.clone())
1151 .ok_or(ObjectsError::NotFound),
1152 }
1153 }
1154
1155 fn target_ref<'a>(
1156 state: &'a State,
1157 target: &ReadTarget,
1158 ) -> Result<&'a BucketState, ObjectsError> {
1159 match target {
1160 ReadTarget::Bucket(reference) => Self::bucket(state, reference),
1161 ReadTarget::Snapshot(reference) => state
1162 .snapshots
1163 .get(&reference.snapshot_id)
1164 .filter(|snapshot| snapshot.reference == *reference)
1165 .map(|snapshot| &snapshot.bucket)
1166 .ok_or(ObjectsError::NotFound),
1167 }
1168 }
1169
1170 fn visible<'a>(
1171 bucket: &'a BucketState,
1172 object_key: &str,
1173 version_id: Option<&str>,
1174 ) -> Result<&'a Version, ObjectsError> {
1175 let versions = bucket
1176 .objects
1177 .get(object_key)
1178 .ok_or(ObjectsError::NotFound)?;
1179 let version = match version_id {
1180 Some(identity) => versions
1181 .iter()
1182 .rev()
1183 .find(|version| version.descriptor.version_id == identity),
1184 None => versions.last(),
1185 }
1186 .ok_or(ObjectsError::NotFound)?;
1187 if version.descriptor.delete_marker {
1188 Err(ObjectsError::NotFound)
1189 } else {
1190 Ok(version)
1191 }
1192 }
1193
1194 fn resolve_get(state: &State, request: &GetRequest) -> Result<ResolvedGet, ObjectsError> {
1195 Self::validate_key(&request.object_key)?;
1196 let bucket = Self::target_ref(state, &request.target)?;
1197 let version = Self::visible(bucket, &request.object_key, request.version_id.as_deref())?;
1198 if request
1199 .if_match
1200 .as_ref()
1201 .is_some_and(|etag| *etag != version.descriptor.etag)
1202 || request
1203 .if_none_match
1204 .as_ref()
1205 .is_some_and(|etag| *etag == version.descriptor.etag)
1206 {
1207 return Err(ObjectsError::PreconditionFailed);
1208 }
1209 let descriptor = version.descriptor.clone();
1210 let body = version.body.clone().ok_or(ObjectsError::NotFound)?;
1211 let (start, end) = match request.range {
1212 None => (0, body.len()),
1213 Some((start, end)) => {
1214 let start =
1215 usize::try_from(start).map_err(|_| ObjectsError::Invalid("invalid range"))?;
1216 let end = usize::try_from(end.unwrap_or(body.len().saturating_sub(1) as u64))
1217 .map_err(|_| ObjectsError::Invalid("invalid range"))?;
1218 if start > end || end >= body.len() {
1219 return Err(ObjectsError::Invalid("invalid range"));
1220 }
1221 (start, end.saturating_add(1))
1222 }
1223 };
1224 if end.saturating_sub(start) as u64 > request.maximum_bytes {
1225 return Err(ObjectsError::Capacity);
1226 }
1227 Ok(ResolvedGet {
1228 descriptor,
1229 body,
1230 start,
1231 end,
1232 })
1233 }
1234
1235 async fn read_resolved_get(resolved: ResolvedGet) -> Result<BufferedObject, ObjectsError> {
1236 let body = resolved
1237 .body
1238 .read_async(resolved.start, resolved.end)
1239 .await?;
1240 Ok(BufferedObject {
1241 version: resolved.descriptor,
1242 body,
1243 })
1244 }
1245
1246 fn condition(current: Option<&Version>, condition: &Option<Condition>) -> bool {
1247 match condition {
1248 None => true,
1249 Some(Condition::IfAbsent) => current.is_none_or(|value| value.descriptor.delete_marker),
1250 Some(Condition::IfMatch(etag)) => current.is_some_and(|value| {
1251 !value.descriptor.delete_marker && value.descriptor.etag == *etag
1252 }),
1253 Some(Condition::IfVersion(identity)) => {
1254 current.is_some_and(|value| value.descriptor.version_id == *identity)
1255 }
1256 }
1257 }
1258
1259 fn descriptor(
1260 state: &mut State,
1261 body_digest: &[u8; 32],
1262 body_length: usize,
1263 metadata: wire::ObjectMetadata,
1264 marker: bool,
1265 ) -> Result<wire::ObjectVersion, ObjectsError> {
1266 let version_id = Self::next(state, "version")?;
1267 let etag = if marker {
1268 String::new()
1269 } else {
1270 format!("\"{}\"", blake3::Hash::from_bytes(*body_digest).to_hex())
1271 };
1272 Ok(wire::ObjectVersion {
1273 version_id,
1274 etag,
1275 size: body_length as u64,
1276 delete_marker: marker,
1277 metadata: Some(metadata),
1278 created_at: Some(Self::timestamp(state)),
1279 })
1280 }
1281
1282 fn prefix_successor(prefix: &str) -> Option<String> {
1283 let mut chars: Vec<char> = prefix.chars().collect();
1284 while let Some(last) = chars.pop() {
1285 let next = u32::from(last) + 1;
1286 let next = char::from_u32(if next == 0xd800 { 0xe000 } else { next });
1287 if let Some(next) = next {
1288 chars.push(next);
1289 return Some(chars.into_iter().collect());
1290 }
1291 }
1292 None
1293 }
1294
1295 fn listing_next(
1296 objects: &OrdMap<String, Vec<wire::ObjectVersion>>,
1297 prefix: &str,
1298 delimiter: Option<&str>,
1299 versions: bool,
1300 cursor: &mut ListingCursor,
1301 ) -> Option<ListingItem> {
1302 while let Some(key) = cursor.key.as_ref() {
1303 let Some((object_key, history)) = objects.range(key.clone()..).next() else {
1304 cursor.key = None;
1305 return None;
1306 };
1307 if !object_key.starts_with(prefix) {
1308 cursor.key = None;
1309 return None;
1310 }
1311 if let Some(delimiter) = delimiter
1312 && let Some(remainder) = object_key.get(prefix.len()..)
1313 && let Some(position) = remainder.find(delimiter)
1314 && let Some(group) = object_key.get(..prefix.len() + position + delimiter.len())
1315 {
1316 cursor.key = Self::prefix_successor(group);
1317 cursor.version_index = 0;
1318 return Some(ListingItem::Prefix(group.to_owned()));
1319 }
1320 if versions {
1321 if let Some(version) = history
1322 .len()
1323 .checked_sub(cursor.version_index.saturating_add(1))
1324 .and_then(|index| history.get(index))
1325 {
1326 cursor.version_index += 1;
1327 if cursor.version_index == history.len() {
1328 cursor.key = Some(format!("{object_key}\0"));
1329 cursor.version_index = 0;
1330 }
1331 return Some(ListingItem::Entry(Box::new(wire::ListEntry {
1332 object_key: object_key.clone(),
1333 version: Some(version.clone()),
1334 })));
1335 }
1336 } else if let Some(version) = history.last()
1337 && !version.delete_marker
1338 {
1339 cursor.key = Some(format!("{object_key}\0"));
1340 cursor.version_index = 0;
1341 return Some(ListingItem::Entry(Box::new(wire::ListEntry {
1342 object_key: object_key.clone(),
1343 version: Some(version.clone()),
1344 })));
1345 }
1346 cursor.key = Some(format!("{object_key}\0"));
1347 cursor.version_index = 0;
1348 }
1349 None
1350 }
1351
1352 fn listing_token(
1353 view_id: &str,
1354 offset: usize,
1355 cursor: &ListingCursor,
1356 ) -> Result<String, ObjectsError> {
1357 let key = cursor.key.as_deref().ok_or(ObjectsError::Unavailable)?;
1358 let payload = format!(
1359 "{view_id}:{offset}:{}:{}",
1360 cursor.version_index,
1361 hex::encode(key.as_bytes())
1362 );
1363 Ok(payload)
1364 }
1365
1366 fn parse_listing_token(token: &str) -> Result<(String, usize, ListingCursor), ObjectsError> {
1367 let mut fields = token.split(':');
1368 let view_id = fields
1369 .next()
1370 .ok_or(ObjectsError::Invalid("invalid continuation"))?;
1371 let offset = fields
1372 .next()
1373 .ok_or(ObjectsError::Invalid("invalid continuation"))?
1374 .parse()
1375 .map_err(|_| ObjectsError::Invalid("invalid continuation"))?;
1376 let version_index = fields
1377 .next()
1378 .ok_or(ObjectsError::Invalid("invalid continuation"))?
1379 .parse()
1380 .map_err(|_| ObjectsError::Invalid("invalid continuation"))?;
1381 let key = fields
1382 .next()
1383 .ok_or(ObjectsError::Invalid("invalid continuation"))?;
1384 if fields.next().is_some() {
1385 return Err(ObjectsError::Invalid("invalid continuation"));
1386 }
1387 let key = String::from_utf8(
1388 hex::decode(key).map_err(|_| ObjectsError::Invalid("invalid continuation"))?,
1389 )
1390 .map_err(|_| ObjectsError::Invalid("invalid continuation"))?;
1391 Ok((
1392 view_id.to_owned(),
1393 offset,
1394 ListingCursor {
1395 key: Some(key),
1396 version_index,
1397 },
1398 ))
1399 }
1400
1401 fn listing_page(
1402 view: &mut ListingView,
1403 view_id: &str,
1404 offset: usize,
1405 page_size: u32,
1406 mut cursor: ListingCursor,
1407 ) -> Result<ProviderListPage, ObjectsError> {
1408 let mut prefetched = view
1409 .prefetched
1410 .take()
1411 .filter(|item| item.offset == offset && item.cursor == cursor);
1412 let mut entries = Vec::new();
1413 let mut common_prefixes = Vec::new();
1414 for _ in 0..page_size {
1415 let item = if let Some(item) = prefetched.take() {
1416 cursor = item.after;
1417 Some(item.item)
1418 } else {
1419 Self::listing_next(
1420 &view.objects,
1421 &view.binding.prefix,
1422 view.binding.delimiter.as_deref(),
1423 view.binding.versions,
1424 &mut cursor,
1425 )
1426 };
1427 let Some(item) = item else {
1428 break;
1429 };
1430 match item {
1431 ListingItem::Entry(value) => entries.push(*value),
1432 ListingItem::Prefix(value) => common_prefixes.push(value),
1433 }
1434 }
1435 let end = offset + entries.len() + common_prefixes.len();
1436 let mut lookahead = cursor.clone();
1437 let continuation = Self::listing_next(
1438 &view.objects,
1439 &view.binding.prefix,
1440 view.binding.delimiter.as_deref(),
1441 view.binding.versions,
1442 &mut lookahead,
1443 )
1444 .map(|item| {
1445 view.prefetched = Some(PrefetchedListingItem {
1446 offset: end,
1447 cursor: cursor.clone(),
1448 item,
1449 after: lookahead,
1450 });
1451 Self::listing_token(view_id, end, &cursor)
1452 })
1453 .transpose()?;
1454 Ok(ProviderListPage {
1455 entries,
1456 common_prefixes,
1457 continuation,
1458 })
1459 }
1460
1461 fn put_fingerprint(request: &PutRequest, body_digest: &[u8; 32]) -> blake3::Hash {
1462 let bucket_fields = Self::reference_fields(&request.bucket);
1463 let condition_bytes = Self::condition_bytes(&request.condition);
1464 let metadata_bytes = Self::metadata_bytes(&request.metadata);
1465 Self::fingerprint(
1466 "put",
1467 &[
1468 bucket_fields[0],
1469 bucket_fields[1],
1470 request.object_key.as_bytes(),
1471 &condition_bytes,
1472 &metadata_bytes,
1473 body_digest,
1474 ],
1475 )
1476 }
1477
1478 #[cfg(feature = "local")]
1479 pub(crate) async fn recorded_put(
1480 &self,
1481 request: &PutRequest,
1482 body_digest: &[u8; 32],
1483 ) -> Option<Result<wire::ObjectVersion, ObjectsError>> {
1484 let key = request.idempotency_key.as_deref()?;
1485 if key.is_empty()
1488 || key.len() > 256
1489 || Self::validate_key(&request.object_key).is_err()
1490 || Self::validate_metadata(&request.metadata).is_err()
1491 {
1492 return None;
1493 }
1494 let fingerprint = Self::put_fingerprint(request, body_digest);
1495 let state = self.state.lock().await;
1496 let prior = state.idempotency.get(key)?;
1497 if prior.fingerprint != fingerprint {
1498 return None;
1499 }
1500 match &prior.outcome {
1501 MutationOutcome::Version(result) => Some(result.clone()),
1502 _ => None,
1503 }
1504 }
1505
1506 async fn put_stored(
1507 &self,
1508 request: PutRequest,
1509 body: StoredBody,
1510 body_digest: [u8; 32],
1511 ) -> Result<wire::ObjectVersion, ObjectsError> {
1512 Self::validate_key(&request.object_key)?;
1513 Self::validate_metadata(&request.metadata)?;
1514 let fingerprint = Self::put_fingerprint(&request, &body_digest);
1515 let PutRequest {
1516 bucket,
1517 object_key,
1518 body: _,
1519 metadata,
1520 condition,
1521 idempotency_key,
1522 } = request;
1523 let mut state = self.state.lock().await;
1524 Self::execute(
1525 &mut state,
1526 idempotency_key.as_deref(),
1527 fingerprint,
1528 |outcome| match outcome {
1529 MutationOutcome::Version(value) => Some(value.clone()),
1530 _ => None,
1531 },
1532 MutationOutcome::Version,
1533 |state| {
1534 let current = Self::bucket(state, &bucket)?
1535 .objects
1536 .get(&object_key)
1537 .and_then(|history| history.last())
1538 .cloned();
1539 if !Self::condition(current.as_ref(), &condition) {
1540 return Err(ObjectsError::PreconditionFailed);
1541 }
1542 let size = body.len();
1543 let additional = state.additional_body_bytes(&body)?;
1544 if size > state.maximum_object_bytes
1545 || state
1546 .bytes
1547 .checked_add(additional)
1548 .is_none_or(|bytes| bytes > state.maximum_bytes)
1549 {
1550 return Err(ObjectsError::Capacity);
1551 }
1552 let descriptor = Self::descriptor(state, &body_digest, size, metadata, false)?;
1553 state.retain_body(&body)?;
1554 Self::bucket_mut(state, &bucket)?.append(
1555 object_key,
1556 Version {
1557 descriptor: descriptor.clone(),
1558 body: Some(body),
1559 },
1560 );
1561 Ok(descriptor)
1562 },
1563 )
1564 }
1565
1566 #[cfg(feature = "local")]
1567 pub(crate) async fn put_external(
1568 &self,
1569 request: PutRequest,
1570 external: ExternalBody,
1571 ) -> Result<wire::ObjectVersion, ObjectsError> {
1572 let ExternalBody {
1573 root,
1574 digest,
1575 length,
1576 location,
1577 } = external;
1578 self.put_stored(
1579 request,
1580 StoredBody::Local {
1581 root: Arc::new(root),
1582 digest,
1583 length,
1584 location,
1585 },
1586 digest,
1587 )
1588 .await
1589 }
1590
1591 async fn upload_part_stored(
1592 &self,
1593 request: StoredPartRequest,
1594 ) -> Result<wire::UploadedPart, ObjectsError> {
1595 let StoredPartRequest {
1596 bucket,
1597 object_key,
1598 upload_id,
1599 part_number,
1600 body,
1601 body_digest,
1602 idempotency_key,
1603 } = request;
1604 if !(1..=limits::MULTIPART_PARTS).contains(&part_number)
1605 || body.len() as u64 > limits::MAX_MULTIPART_PART_BYTES
1606 {
1607 return Err(ObjectsError::Invalid("invalid multipart part"));
1608 }
1609 let bucket_fields = Self::reference_fields(&bucket);
1610 let part_number_bytes = part_number.to_le_bytes();
1611 let fingerprint = Self::fingerprint(
1612 "upload-part",
1613 &[
1614 bucket_fields[0],
1615 bucket_fields[1],
1616 object_key.as_bytes(),
1617 upload_id.as_bytes(),
1618 &part_number_bytes,
1619 &body_digest,
1620 ],
1621 );
1622 let mut state = self.state.lock().await;
1623 Self::execute(
1624 &mut state,
1625 idempotency_key.as_deref(),
1626 fingerprint,
1627 |outcome| match outcome {
1628 MutationOutcome::Part(value) => Some(value.clone()),
1629 _ => None,
1630 },
1631 MutationOutcome::Part,
1632 |state| {
1633 let existing_body = state
1634 .multiparts
1635 .get(&upload_id)
1636 .filter(|upload| upload.bucket == bucket && upload.object_key == object_key)
1637 .ok_or(ObjectsError::NotFound)?
1638 .parts
1639 .get(&part_number)
1640 .map(|(_, body)| body.clone());
1641 if let Some(existing) = &existing_body {
1642 state.release_body(existing)?;
1643 }
1644 let additional = state.additional_body_bytes(&body)?;
1645 if state
1646 .bytes
1647 .checked_add(additional)
1648 .is_none_or(|bytes| bytes > state.maximum_bytes)
1649 {
1650 if let Some(existing) = &existing_body {
1651 state.retain_body(existing)?;
1652 }
1653 return Err(ObjectsError::Capacity);
1654 }
1655 let part = wire::UploadedPart {
1656 part_number,
1657 etag: format!("\"{}\"", blake3::Hash::from_bytes(body_digest).to_hex()),
1658 size: body.len() as u64,
1659 };
1660 state
1661 .multiparts
1662 .get_mut(&upload_id)
1663 .ok_or(ObjectsError::NotFound)?
1664 .parts
1665 .insert(part_number, (part.clone(), body));
1666 let admitted = state
1667 .multiparts
1668 .get(&upload_id)
1669 .and_then(|upload| upload.parts.get(&part_number))
1670 .map(|(_, body)| body.clone())
1671 .ok_or(ObjectsError::Unavailable)?;
1672 state.retain_body(&admitted)?;
1673 Ok(part)
1674 },
1675 )
1676 }
1677
1678 #[cfg(feature = "local")]
1679 pub(crate) async fn upload_part_external(
1680 &self,
1681 bucket: wire::BucketRef,
1682 object_key: String,
1683 upload_id: String,
1684 part_number: u32,
1685 external: ExternalBody,
1686 idempotency_key: Option<String>,
1687 ) -> Result<wire::UploadedPart, ObjectsError> {
1688 let ExternalBody {
1689 root,
1690 digest,
1691 length,
1692 location,
1693 } = external;
1694 self.upload_part_stored(StoredPartRequest {
1695 bucket,
1696 object_key,
1697 upload_id,
1698 part_number,
1699 body: StoredBody::Local {
1700 root: Arc::new(root),
1701 digest,
1702 length,
1703 location,
1704 },
1705 body_digest: digest,
1706 idempotency_key,
1707 })
1708 .await
1709 }
1710}
1711
1712#[async_trait]
1713impl ObjectsProvider for MemoryObjects {
1714 async fn create_bucket(
1715 &self,
1716 name: String,
1717 idempotency_key: Option<String>,
1718 ) -> Result<wire::Bucket, ObjectsError> {
1719 Self::validate_bucket_name(&name)?;
1720 let fingerprint = Self::fingerprint("create-bucket", &[name.as_bytes()]);
1721 let mut state = self.state.lock().await;
1722 Self::execute(
1723 &mut state,
1724 idempotency_key.as_deref(),
1725 fingerprint,
1726 |outcome| match outcome {
1727 MutationOutcome::Bucket(value) => Some(value.clone()),
1728 _ => None,
1729 },
1730 MutationOutcome::Bucket,
1731 |state| {
1732 if state.names.contains_key(&name) {
1733 return Err(ObjectsError::AlreadyExists);
1734 }
1735 let bucket_id = Self::next(state, "bucket")?;
1736 let reference = wire::BucketRef {
1737 bucket_id: bucket_id.clone(),
1738 name: name.clone(),
1739 };
1740 let bucket = BucketState {
1741 reference: reference.clone(),
1742 created_at: Self::timestamp(state),
1743 objects: OrdMap::new(),
1744 listing: OrdMap::new(),
1745 };
1746 state.names.insert(name, bucket_id.clone());
1747 state.buckets.insert(bucket_id, bucket.clone());
1748 Ok(wire::Bucket {
1749 bucket: Some(reference),
1750 created_at: Some(bucket.created_at),
1751 })
1752 },
1753 )
1754 }
1755
1756 async fn head_bucket(&self, bucket: &wire::BucketRef) -> Result<wire::Bucket, ObjectsError> {
1757 let state = self.state.lock().await;
1758 let bucket = Self::bucket(&state, bucket)?;
1759 Ok(wire::Bucket {
1760 bucket: Some(bucket.reference.clone()),
1761 created_at: Some(bucket.created_at),
1762 })
1763 }
1764
1765 async fn delete_bucket(
1766 &self,
1767 bucket: &wire::BucketRef,
1768 idempotency_key: Option<String>,
1769 ) -> Result<bool, ObjectsError> {
1770 let fields = Self::reference_fields(bucket);
1771 let fingerprint = Self::fingerprint("delete-bucket", &fields);
1772 let mut state = self.state.lock().await;
1773 Self::execute(
1774 &mut state,
1775 idempotency_key.as_deref(),
1776 fingerprint,
1777 |outcome| match outcome {
1778 MutationOutcome::Boolean(value) => Some(value.clone()),
1779 _ => None,
1780 },
1781 MutationOutcome::Boolean,
1782 |state| {
1783 let existing = Self::bucket(state, bucket)?.clone();
1784 if existing.objects.values().any(|history| !history.is_empty())
1785 || state
1786 .multiparts
1787 .values()
1788 .any(|upload| upload.bucket == *bucket)
1789 {
1790 return Err(ObjectsError::PreconditionFailed);
1791 }
1792 state.buckets.remove(&bucket.bucket_id);
1793 state.names.remove(&bucket.name);
1794 Ok(true)
1795 },
1796 )
1797 }
1798
1799 async fn put(&self, request: PutRequest) -> Result<wire::ObjectVersion, ObjectsError> {
1800 let digest = *blake3::hash(&request.body).as_bytes();
1801 let body = StoredBody::memory(request.body.clone());
1802 self.put_stored(request, body, digest).await
1803 }
1804
1805 async fn get(&self, request: GetRequest) -> Result<BufferedObject, ObjectsError> {
1806 let state = self.state.lock().await;
1807 let resolved = Self::resolve_get(&state, &request)?;
1808 drop(state);
1809 Self::read_resolved_get(resolved).await
1810 }
1811
1812 async fn get_batch(
1813 &self,
1814 requests: Vec<GetRequest>,
1815 ) -> Vec<Result<BufferedObject, ObjectsError>> {
1816 let state = self.state.lock().await;
1817 let resolved = requests
1818 .iter()
1819 .map(|request| Self::resolve_get(&state, request))
1820 .collect::<Vec<_>>();
1821 drop(state);
1822 collect_ordered_bounded(
1823 resolved.into_iter().map(|result| async move {
1824 match result {
1825 Ok(value) => Self::read_resolved_get(value).await,
1826 Err(error) => Err(error),
1827 }
1828 }),
1829 MAX_CONCURRENT_BATCH_READS,
1830 )
1831 .await
1832 }
1833
1834 async fn delete(
1835 &self,
1836 bucket: wire::BucketRef,
1837 object_key: String,
1838 version_id: Option<String>,
1839 condition: Option<Condition>,
1840 idempotency_key: Option<String>,
1841 ) -> Result<DeleteResult, ObjectsError> {
1842 Self::validate_key(&object_key)?;
1843 let bucket_fields = Self::reference_fields(&bucket);
1844 let condition_bytes = Self::condition_bytes(&condition);
1845 let fingerprint = Self::fingerprint(
1846 "delete",
1847 &[
1848 bucket_fields[0],
1849 bucket_fields[1],
1850 object_key.as_bytes(),
1851 version_id.as_deref().unwrap_or_default().as_bytes(),
1852 &condition_bytes,
1853 ],
1854 );
1855 let mut state = self.state.lock().await;
1856 Self::execute(
1857 &mut state,
1858 idempotency_key.as_deref(),
1859 fingerprint,
1860 |outcome| match outcome {
1861 MutationOutcome::Delete(value) => Some(value.clone()),
1862 _ => None,
1863 },
1864 MutationOutcome::Delete,
1865 |state| {
1866 let current = Self::bucket(state, &bucket)?
1867 .objects
1868 .get(&object_key)
1869 .and_then(|history| history.last())
1870 .cloned();
1871 if !Self::condition(current.as_ref(), &condition) {
1872 return Err(ObjectsError::PreconditionFailed);
1873 }
1874 if let Some(identity) = version_id {
1875 let Some(removed) =
1876 Self::bucket_mut(state, &bucket)?.remove_exact(&object_key, &identity)?
1877 else {
1878 return Ok(DeleteResult {
1879 existed: false,
1880 marker: None,
1881 });
1882 };
1883 if let Some(body) = removed.body {
1884 state.release_body(&body)?;
1885 }
1886 Ok(DeleteResult {
1887 existed: true,
1888 marker: None,
1889 })
1890 } else {
1891 let marker = Self::descriptor(
1892 state,
1893 blake3::hash(&[]).as_bytes(),
1894 0,
1895 wire::ObjectMetadata::default(),
1896 true,
1897 )?;
1898 Self::bucket_mut(state, &bucket)?.append(
1899 object_key,
1900 Version {
1901 descriptor: marker.clone(),
1902 body: None,
1903 },
1904 );
1905 Ok(DeleteResult {
1906 existed: true,
1907 marker: Some(marker),
1908 })
1909 }
1910 },
1911 )
1912 }
1913
1914 async fn list(
1915 &self,
1916 target: ReadTarget,
1917 prefix: String,
1918 delimiter: Option<String>,
1919 versions: bool,
1920 page_size: u32,
1921 continuation: Option<String>,
1922 ) -> Result<ProviderListPage, ObjectsError> {
1923 if page_size == 0
1924 || page_size > limits::LIST_PAGE_ENTRIES
1925 || delimiter.as_ref().is_some_and(String::is_empty)
1926 {
1927 return Err(ObjectsError::Invalid("invalid listing"));
1928 }
1929 let mut state = self.state.lock().await;
1930 if continuation.is_none() {
1931 let now = Instant::now();
1932 state.listings.retain(|_, view| now < view.expires_at);
1933 }
1934 let binding = ListingBinding {
1935 target: target.clone(),
1936 prefix: prefix.clone(),
1937 delimiter: delimiter.clone(),
1938 versions,
1939 };
1940 let fresh = continuation.is_none();
1941 let (view_id, offset, cursor) = if let Some(token) = continuation {
1942 Self::parse_listing_token(&token)?
1943 } else {
1944 let objects = Self::target_ref(&state, &target)?.listing.clone();
1945 let sequence = Self::next(&mut state, "listing")?;
1946 let view_id = format!("{}-{sequence}", state.listing_epoch);
1947 state.listings.insert(
1948 view_id.clone(),
1949 ListingView {
1950 binding: binding.clone(),
1951 objects,
1952 expires_at: Instant::now() + Duration::from_secs(limits::LISTING_VIEW_SECONDS),
1953 prefetched: None,
1954 },
1955 );
1956 (
1957 view_id,
1958 0,
1959 ListingCursor {
1960 key: Some(prefix.clone()),
1961 version_index: 0,
1962 },
1963 )
1964 };
1965 if state
1966 .listings
1967 .get(&view_id)
1968 .is_some_and(|view| Instant::now() >= view.expires_at)
1969 {
1970 state.listings.remove(&view_id);
1971 return Err(ObjectsError::Invalid("invalid continuation"));
1972 }
1973 let view = state
1974 .listings
1975 .get_mut(&view_id)
1976 .filter(|view| view.binding == binding)
1977 .ok_or(ObjectsError::Invalid("invalid continuation"))?;
1978 let page = Self::listing_page(view, &view_id, offset, page_size, cursor)?;
1979 if fresh {
1980 if page.continuation.is_none() {
1981 state.listings.remove(&view_id);
1982 } else if state.listings.len() > MAX_LISTING_VIEWS {
1983 state.listings.remove(&view_id);
1984 return Err(ObjectsError::Capacity);
1985 }
1986 }
1987 Ok(page)
1988 }
1989
1990 async fn snapshot(
1991 &self,
1992 bucket: wire::BucketRef,
1993 idempotency_key: Option<String>,
1994 ) -> Result<wire::Snapshot, ObjectsError> {
1995 let fields = Self::reference_fields(&bucket);
1996 let fingerprint = Self::fingerprint("snapshot", &fields);
1997 let mut state = self.state.lock().await;
1998 Self::execute(
1999 &mut state,
2000 idempotency_key.as_deref(),
2001 fingerprint,
2002 |outcome| match outcome {
2003 MutationOutcome::Snapshot(value) => Some(value.clone()),
2004 _ => None,
2005 },
2006 MutationOutcome::Snapshot,
2007 |state| {
2008 let source = Self::bucket(state, &bucket)?.clone();
2009 state.retain_bucket(&source)?;
2010 let snapshot_id = Self::next(state, "snapshot")?;
2011 let reference = wire::SnapshotRef {
2012 snapshot_id: snapshot_id.clone(),
2013 source_bucket_id: bucket.bucket_id,
2014 };
2015 let created_at = Self::timestamp(state);
2016 state.snapshots.insert(
2017 snapshot_id,
2018 SnapshotState {
2019 reference: reference.clone(),
2020 bucket: source,
2021 },
2022 );
2023 Ok(wire::Snapshot {
2024 snapshot: Some(reference),
2025 created_at: Some(created_at),
2026 })
2027 },
2028 )
2029 }
2030
2031 async fn destroy_snapshot(
2032 &self,
2033 snapshot: wire::SnapshotRef,
2034 idempotency_key: Option<String>,
2035 ) -> Result<bool, ObjectsError> {
2036 let fingerprint = Self::fingerprint(
2037 "destroy-snapshot",
2038 &[
2039 snapshot.snapshot_id.as_bytes(),
2040 snapshot.source_bucket_id.as_bytes(),
2041 ],
2042 );
2043 let mut state = self.state.lock().await;
2044 Self::execute(
2045 &mut state,
2046 idempotency_key.as_deref(),
2047 fingerprint,
2048 |outcome| match outcome {
2049 MutationOutcome::Boolean(value) => Some(value.clone()),
2050 _ => None,
2051 },
2052 MutationOutcome::Boolean,
2053 |state| {
2054 let exists = state
2055 .snapshots
2056 .get(&snapshot.snapshot_id)
2057 .is_some_and(|value| value.reference == snapshot);
2058 if !exists {
2059 return Ok(false);
2060 }
2061 let removed = state
2062 .snapshots
2063 .remove(&snapshot.snapshot_id)
2064 .ok_or(ObjectsError::Unavailable)?;
2065 state.release_bucket(&removed.bucket)?;
2066 Ok(true)
2067 },
2068 )
2069 }
2070
2071 async fn fork(
2072 &self,
2073 source: ReadTarget,
2074 destination_name: String,
2075 idempotency_key: Option<String>,
2076 ) -> Result<wire::Bucket, ObjectsError> {
2077 let source_identity = match &source {
2078 ReadTarget::Bucket(value) => format!("bucket:{}:{}", value.bucket_id, value.name),
2079 ReadTarget::Snapshot(value) => {
2080 format!("snapshot:{}:{}", value.snapshot_id, value.source_bucket_id)
2081 }
2082 };
2083 let fingerprint = Self::fingerprint(
2084 "fork",
2085 &[source_identity.as_bytes(), destination_name.as_bytes()],
2086 );
2087 let mut state = self.state.lock().await;
2088 Self::execute(
2089 &mut state,
2090 idempotency_key.as_deref(),
2091 fingerprint,
2092 |outcome| match outcome {
2093 MutationOutcome::Bucket(value) => Some(value.clone()),
2094 _ => None,
2095 },
2096 MutationOutcome::Bucket,
2097 |state| {
2098 if state.names.contains_key(&destination_name) {
2099 return Err(ObjectsError::AlreadyExists);
2100 }
2101 let source = Self::target(state, &source)?;
2102 state.retain_bucket(&source)?;
2103 let bucket_id = Self::next(state, "bucket")?;
2104 let reference = wire::BucketRef {
2105 bucket_id: bucket_id.clone(),
2106 name: destination_name.clone(),
2107 };
2108 let created_at = Self::timestamp(state);
2109 state.names.insert(destination_name, bucket_id.clone());
2110 state.buckets.insert(
2111 bucket_id,
2112 BucketState {
2113 reference: reference.clone(),
2114 created_at,
2115 objects: source.objects,
2116 listing: source.listing,
2117 },
2118 );
2119 Ok(wire::Bucket {
2120 bucket: Some(reference),
2121 created_at: Some(created_at),
2122 })
2123 },
2124 )
2125 }
2126
2127 async fn create_multipart(
2128 &self,
2129 bucket: wire::BucketRef,
2130 object_key: String,
2131 metadata: wire::ObjectMetadata,
2132 condition: Option<Condition>,
2133 idempotency_key: Option<String>,
2134 ) -> Result<wire::MultipartUpload, ObjectsError> {
2135 Self::validate_key(&object_key)?;
2136 Self::validate_metadata(&metadata)?;
2137 let bucket_fields = Self::reference_fields(&bucket);
2138 let metadata_bytes = Self::metadata_bytes(&metadata);
2139 let condition_bytes = Self::condition_bytes(&condition);
2140 let fingerprint = Self::fingerprint(
2141 "create-multipart",
2142 &[
2143 bucket_fields[0],
2144 bucket_fields[1],
2145 object_key.as_bytes(),
2146 &metadata_bytes,
2147 &condition_bytes,
2148 ],
2149 );
2150 let mut state = self.state.lock().await;
2151 Self::execute(
2152 &mut state,
2153 idempotency_key.as_deref(),
2154 fingerprint,
2155 |outcome| match outcome {
2156 MutationOutcome::Multipart(value) => Some(value.clone()),
2157 _ => None,
2158 },
2159 MutationOutcome::Multipart,
2160 |state| {
2161 Self::bucket(state, &bucket)?;
2162 let upload_id = Self::next(state, "upload")?;
2163 state.multiparts.insert(
2164 upload_id.clone(),
2165 MultipartState {
2166 bucket,
2167 object_key,
2168 metadata,
2169 condition,
2170 parts: BTreeMap::new(),
2171 },
2172 );
2173 Ok(wire::MultipartUpload { upload_id })
2174 },
2175 )
2176 }
2177
2178 async fn upload_part(
2179 &self,
2180 bucket: wire::BucketRef,
2181 object_key: String,
2182 upload_id: String,
2183 part_number: u32,
2184 body: bytes::Bytes,
2185 idempotency_key: Option<String>,
2186 ) -> Result<wire::UploadedPart, ObjectsError> {
2187 let digest = *blake3::hash(&body).as_bytes();
2188 self.upload_part_stored(StoredPartRequest {
2189 bucket,
2190 object_key,
2191 upload_id,
2192 part_number,
2193 body: StoredBody::memory(body),
2194 body_digest: digest,
2195 idempotency_key,
2196 })
2197 .await
2198 }
2199
2200 async fn list_parts(
2201 &self,
2202 bucket: wire::BucketRef,
2203 object_key: String,
2204 upload_id: String,
2205 ) -> Result<Vec<wire::UploadedPart>, ObjectsError> {
2206 let state = self.state.lock().await;
2207 let upload = state
2208 .multiparts
2209 .get(&upload_id)
2210 .filter(|upload| upload.bucket == bucket && upload.object_key == object_key)
2211 .ok_or(ObjectsError::NotFound)?;
2212 Ok(upload
2213 .parts
2214 .values()
2215 .map(|(part, _)| part.clone())
2216 .collect())
2217 }
2218
2219 async fn complete_multipart(
2220 &self,
2221 bucket: wire::BucketRef,
2222 object_key: String,
2223 upload_id: String,
2224 parts: Vec<wire::UploadedPart>,
2225 idempotency_key: Option<String>,
2226 ) -> Result<wire::ObjectVersion, ObjectsError> {
2227 if parts.is_empty() || parts.len() > limits::MULTIPART_PARTS as usize {
2228 return Err(ObjectsError::Invalid("invalid multipart completion"));
2229 }
2230 let bucket_fields = Self::reference_fields(&bucket);
2231 let mut part_bytes = Vec::new();
2232 for part in &parts {
2233 part_bytes.extend_from_slice(&part.part_number.to_le_bytes());
2234 part_bytes.extend_from_slice(&(part.etag.len() as u64).to_le_bytes());
2235 part_bytes.extend_from_slice(part.etag.as_bytes());
2236 part_bytes.extend_from_slice(&part.size.to_le_bytes());
2237 }
2238 let fingerprint = Self::fingerprint(
2239 "complete-multipart",
2240 &[
2241 bucket_fields[0],
2242 bucket_fields[1],
2243 object_key.as_bytes(),
2244 upload_id.as_bytes(),
2245 &part_bytes,
2246 ],
2247 );
2248 let mut state = self.state.lock().await;
2249 Self::execute(
2250 &mut state,
2251 idempotency_key.as_deref(),
2252 fingerprint,
2253 |outcome| match outcome {
2254 MutationOutcome::Version(value) => Some(value.clone()),
2255 _ => None,
2256 },
2257 MutationOutcome::Version,
2258 |state| {
2259 let upload = state
2260 .multiparts
2261 .remove(&upload_id)
2262 .filter(|upload| upload.bucket == bucket && upload.object_key == object_key)
2263 .ok_or(ObjectsError::NotFound)?;
2264 let exact = upload.parts.values().map(|(part, _)| part).eq(parts.iter());
2265 let sizes_valid = parts.iter().enumerate().all(|(index, part)| {
2266 index + 1 == parts.len() || part.size >= limits::MIN_MULTIPART_PART_BYTES
2267 });
2268 if !exact || !sizes_valid {
2269 state.multiparts.insert(upload_id, upload);
2270 return Err(ObjectsError::Invalid("multipart receipts do not match"));
2271 }
2272 let current = Self::bucket(state, &bucket)?
2273 .objects
2274 .get(&object_key)
2275 .and_then(|history| history.last())
2276 .cloned();
2277 if !Self::condition(current.as_ref(), &upload.condition) {
2278 state.multiparts.insert(upload_id, upload);
2279 return Err(ObjectsError::PreconditionFailed);
2280 }
2281 let body_length =
2282 match upload.parts.values().try_fold(0usize, |total, (_, body)| {
2283 total.checked_add(body.len()).ok_or(ObjectsError::Capacity)
2284 }) {
2285 Ok(body_length) => body_length,
2286 Err(error) => {
2287 state.multiparts.insert(upload_id, upload);
2288 return Err(error);
2289 }
2290 };
2291 if body_length > state.maximum_object_bytes {
2292 state.multiparts.insert(upload_id, upload);
2293 return Err(ObjectsError::Capacity);
2294 }
2295 let bodies = upload
2296 .parts
2297 .into_values()
2298 .map(|(_, body)| body)
2299 .collect::<Vec<_>>();
2300 let mut hasher = blake3::Hasher::new();
2301 for body in &bodies {
2302 body.update_hash(&mut hasher)?;
2303 }
2304 let body_digest = *hasher.finalize().as_bytes();
2305 let descriptor =
2306 Self::descriptor(state, &body_digest, body_length, upload.metadata, false)?;
2307 Self::bucket_mut(state, &bucket)?.append(
2308 object_key,
2309 Version {
2310 descriptor: descriptor.clone(),
2311 body: Some(StoredBody::Composite {
2312 parts: bodies.into(),
2313 length: body_length,
2314 }),
2315 },
2316 );
2317 Ok(descriptor)
2318 },
2319 )
2320 }
2321
2322 async fn abort_multipart(
2323 &self,
2324 bucket: wire::BucketRef,
2325 object_key: String,
2326 upload_id: String,
2327 idempotency_key: Option<String>,
2328 ) -> Result<bool, ObjectsError> {
2329 let bucket_fields = Self::reference_fields(&bucket);
2330 let fingerprint = Self::fingerprint(
2331 "abort-multipart",
2332 &[
2333 bucket_fields[0],
2334 bucket_fields[1],
2335 object_key.as_bytes(),
2336 upload_id.as_bytes(),
2337 ],
2338 );
2339 let mut state = self.state.lock().await;
2340 Self::execute(
2341 &mut state,
2342 idempotency_key.as_deref(),
2343 fingerprint,
2344 |outcome| match outcome {
2345 MutationOutcome::Boolean(value) => Some(value.clone()),
2346 _ => None,
2347 },
2348 MutationOutcome::Boolean,
2349 |state| {
2350 let matches = state.multiparts.get(&upload_id).is_some_and(|upload| {
2351 upload.bucket == bucket && upload.object_key == object_key
2352 });
2353 if !matches {
2354 return Ok(false);
2355 }
2356 if let Some(upload) = state.multiparts.remove(&upload_id) {
2357 for (_, body) in upload.parts.values() {
2358 state.release_body(body)?;
2359 }
2360 }
2361 Ok(true)
2362 },
2363 )
2364 }
2365}
2366
2367#[cfg(test)]
2368mod tests {
2369 use super::*;
2370 use std::collections::BTreeSet;
2371 use std::sync::atomic::{AtomicUsize, Ordering};
2372
2373 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2374 async fn bounded_collection_limits_concurrency_and_restores_order() {
2375 const LIMIT: usize = 3;
2376 let active = Arc::new(AtomicUsize::new(0));
2377 let peak = Arc::new(AtomicUsize::new(0));
2378 let barrier = Arc::new(tokio::sync::Barrier::new(LIMIT));
2379 let futures = (0..11).map(|index| {
2380 let active = Arc::clone(&active);
2381 let peak = Arc::clone(&peak);
2382 let barrier = Arc::clone(&barrier);
2383 async move {
2384 let now = active.fetch_add(1, Ordering::SeqCst) + 1;
2385 peak.fetch_max(now, Ordering::SeqCst);
2386 if index < LIMIT {
2387 barrier.wait().await;
2388 }
2389 tokio::task::yield_now().await;
2390 active.fetch_sub(1, Ordering::SeqCst);
2391 10 - index
2392 }
2393 });
2394
2395 let results = collect_ordered_bounded(futures, LIMIT).await;
2396
2397 assert_eq!(peak.load(Ordering::SeqCst), LIMIT);
2398 assert_eq!(active.load(Ordering::SeqCst), 0);
2399 assert_eq!(results, (0..11).map(|index| 10 - index).collect::<Vec<_>>());
2400 }
2401
2402 #[tokio::test]
2403 async fn get_batch_preserves_order_and_independent_failures() {
2404 let (store, bucket) = MemoryObjects::with_default_bucket();
2405 put(&store, &bucket, "first", b"abc", None).await;
2406 put(&store, &bucket, "second", b"defg", None).await;
2407 let requests = ["second", "missing", "first"]
2408 .into_iter()
2409 .map(|object_key| GetRequest {
2410 target: ReadTarget::Bucket(bucket.clone()),
2411 object_key: object_key.to_owned(),
2412 version_id: None,
2413 range: None,
2414 if_match: None,
2415 if_none_match: None,
2416 maximum_bytes: 4,
2417 })
2418 .collect();
2419 let results = store.get_batch(requests).await;
2420 assert_eq!(results.len(), 3);
2421 assert_eq!(
2422 results
2423 .first()
2424 .and_then(|value| value.as_ref().ok())
2425 .map(|value| &value.body[..]),
2426 Some(b"defg".as_slice())
2427 );
2428 assert!(matches!(results.get(1), Some(Err(ObjectsError::NotFound))));
2429 assert_eq!(
2430 results
2431 .get(2)
2432 .and_then(|value| value.as_ref().ok())
2433 .map(|value| &value.body[..]),
2434 Some(b"abc".as_slice())
2435 );
2436 }
2437
2438 #[tokio::test]
2439 async fn listing_continuation_is_bound_to_exact_query() {
2440 let (store, bucket) = MemoryObjects::with_default_bucket();
2441 put(&store, &bucket, "a|b1", b"first", None).await;
2442 put(&store, &bucket, "a|b2", b"second", None).await;
2443 let first = store
2444 .list(
2445 ReadTarget::Bucket(bucket.clone()),
2446 "a|b".into(),
2447 Some("c".into()),
2448 false,
2449 1,
2450 None,
2451 )
2452 .await
2453 .unwrap_or_else(|_| unreachable!());
2454 assert_eq!(
2455 store
2456 .list(
2457 ReadTarget::Bucket(bucket),
2458 "a".into(),
2459 Some("b|c".into()),
2460 false,
2461 1,
2462 first.continuation,
2463 )
2464 .await,
2465 Err(ObjectsError::Invalid("invalid continuation"))
2466 );
2467 }
2468
2469 #[tokio::test]
2470 async fn listing_views_are_bounded_and_do_not_hold_bodies() {
2471 let (store, bucket) = MemoryObjects::with_default_bucket();
2472 let first = put(&store, &bucket, "a", b"payload", None).await;
2473 put(&store, &bucket, "b", b"other", None).await;
2474 let mut oldest = None;
2475 for index in 0..MAX_LISTING_VIEWS {
2476 let page = store
2477 .list(
2478 ReadTarget::Bucket(bucket.clone()),
2479 String::new(),
2480 None,
2481 false,
2482 1,
2483 None,
2484 )
2485 .await
2486 .unwrap_or_else(|_| unreachable!());
2487 if index == 0 {
2488 oldest = page.continuation;
2489 }
2490 }
2491 assert_eq!(store.state.lock().await.listings.len(), MAX_LISTING_VIEWS);
2492 assert_eq!(
2493 store
2494 .list(
2495 ReadTarget::Bucket(bucket.clone()),
2496 String::new(),
2497 None,
2498 false,
2499 1,
2500 None,
2501 )
2502 .await,
2503 Err(ObjectsError::Capacity)
2504 );
2505 assert_eq!(store.state.lock().await.listings.len(), MAX_LISTING_VIEWS);
2506 let continuation = store
2507 .list(
2508 ReadTarget::Bucket(bucket.clone()),
2509 String::new(),
2510 None,
2511 false,
2512 1,
2513 oldest,
2514 )
2515 .await
2516 .unwrap_or_else(|_| unreachable!());
2517 assert_eq!(
2518 continuation
2519 .entries
2520 .first()
2521 .map(|entry| entry.object_key.as_str()),
2522 Some("b")
2523 );
2524 store
2525 .delete(
2526 bucket.clone(),
2527 "a".into(),
2528 Some(first.version_id),
2529 None,
2530 None,
2531 )
2532 .await
2533 .unwrap_or_else(|_| unreachable!());
2534 assert_eq!(store.state.lock().await.bytes, b"other".len());
2535 let one_page = store
2536 .list(
2537 ReadTarget::Bucket(bucket),
2538 String::new(),
2539 None,
2540 false,
2541 2,
2542 None,
2543 )
2544 .await
2545 .unwrap_or_else(|_| unreachable!());
2546 assert!(one_page.continuation.is_none());
2547 assert_eq!(store.state.lock().await.listings.len(), MAX_LISTING_VIEWS);
2548 }
2549
2550 #[tokio::test]
2551 async fn expired_listing_continuation_is_rejected_and_reaped() {
2552 let (store, bucket) = MemoryObjects::with_default_bucket();
2553 put(&store, &bucket, "a", b"a", None).await;
2554 put(&store, &bucket, "b", b"b", None).await;
2555 let token = store
2556 .list(
2557 ReadTarget::Bucket(bucket.clone()),
2558 String::new(),
2559 None,
2560 false,
2561 1,
2562 None,
2563 )
2564 .await
2565 .unwrap_or_else(|_| unreachable!())
2566 .continuation;
2567 {
2568 let mut state = store.state.lock().await;
2569 let view = state
2570 .listings
2571 .values_mut()
2572 .next()
2573 .unwrap_or_else(|| unreachable!());
2574 view.expires_at = Instant::now();
2575 }
2576 assert_eq!(
2577 store
2578 .list(
2579 ReadTarget::Bucket(bucket),
2580 String::new(),
2581 None,
2582 false,
2583 1,
2584 token
2585 )
2586 .await,
2587 Err(ObjectsError::Invalid("invalid continuation"))
2588 );
2589 assert!(store.state.lock().await.listings.is_empty());
2590 }
2591
2592 #[tokio::test]
2593 async fn long_single_entry_pagination_keeps_one_view_and_bounded_tokens() {
2594 let (store, bucket) = MemoryObjects::with_default_bucket();
2595 for index in 0..2_000 {
2596 put(&store, &bucket, &format!("key-{index:04}"), b"x", None).await;
2597 }
2598 let mut continuation = None;
2599 for expected in 0..2_000 {
2600 let page = store
2601 .list(
2602 ReadTarget::Bucket(bucket.clone()),
2603 String::new(),
2604 None,
2605 false,
2606 1,
2607 continuation,
2608 )
2609 .await
2610 .unwrap_or_else(|_| unreachable!());
2611 assert_eq!(
2612 page.entries.first().map(|entry| entry.object_key.as_str()),
2613 Some(format!("key-{expected:04}").as_str())
2614 );
2615 continuation = page.continuation;
2616 let state = store.state.lock().await;
2617 assert_eq!(state.listings.len(), 1);
2618 let view = state
2619 .listings
2620 .values()
2621 .next()
2622 .unwrap_or_else(|| unreachable!());
2623 assert_eq!(view.objects.len(), 2_000);
2624 assert_eq!(view.prefetched.is_some(), expected < 1_999);
2625 assert!(continuation.as_ref().is_none_or(|token| token.len() < 256));
2626 }
2627 assert!(continuation.is_none());
2628 }
2629
2630 #[tokio::test]
2631 async fn listing_prefetch_preserves_replay_and_page_size_changes() {
2632 let (store, bucket) = MemoryObjects::with_default_bucket();
2633 for key in ["a", "b", "c", "d"] {
2634 put(&store, &bucket, key, b"x", None).await;
2635 }
2636 let first = store
2637 .list(
2638 ReadTarget::Bucket(bucket.clone()),
2639 String::new(),
2640 None,
2641 false,
2642 1,
2643 None,
2644 )
2645 .await
2646 .unwrap_or_else(|_| unreachable!());
2647 let token = first.continuation.unwrap_or_else(|| unreachable!());
2648 let two = store
2649 .list(
2650 ReadTarget::Bucket(bucket.clone()),
2651 String::new(),
2652 None,
2653 false,
2654 2,
2655 Some(token.clone()),
2656 )
2657 .await
2658 .unwrap_or_else(|_| unreachable!());
2659 let replay = store
2660 .list(
2661 ReadTarget::Bucket(bucket),
2662 String::new(),
2663 None,
2664 false,
2665 1,
2666 Some(token),
2667 )
2668 .await
2669 .unwrap_or_else(|_| unreachable!());
2670 assert_eq!(
2671 two.entries
2672 .iter()
2673 .map(|entry| entry.object_key.as_str())
2674 .collect::<Vec<_>>(),
2675 ["b", "c"]
2676 );
2677 assert_eq!(
2678 replay
2679 .entries
2680 .first()
2681 .map(|entry| entry.object_key.as_str()),
2682 Some("b")
2683 );
2684 }
2685
2686 #[allow(
2687 clippy::too_many_lines,
2688 reason = "one deterministic matrix compares every paginated mode with an independent model"
2689 )]
2690 #[tokio::test]
2691 async fn paged_listing_matches_independent_sorted_model() {
2692 let (store, bucket) = MemoryObjects::with_default_bucket();
2693 let keys = [
2694 "a",
2695 "a/",
2696 "a/child",
2697 "a//child",
2698 "a/child/next",
2699 "a:one",
2700 "a::two",
2701 "b",
2702 "é/child",
2703 "\u{d7ff}/child",
2704 "\u{e000}/child",
2705 "\u{10ffff}/child",
2706 ];
2707 let mut history = BTreeMap::<String, Vec<wire::ObjectVersion>>::new();
2708 for key in keys {
2709 let first = put(&store, &bucket, key, b"first", None).await;
2710 let second = put(&store, &bucket, key, b"second", None).await;
2711 history.insert(key.to_owned(), vec![first, second]);
2712 }
2713 let deleted = store
2714 .delete(bucket.clone(), "b".into(), None, None, None)
2715 .await
2716 .unwrap_or_else(|_| unreachable!())
2717 .marker
2718 .unwrap_or_else(|| unreachable!());
2719 history
2720 .get_mut("b")
2721 .unwrap_or_else(|| unreachable!())
2722 .push(deleted);
2723 for prefix in ["", "a", "a/", "é", "\u{d7ff}", "\u{10ffff}"] {
2724 for delimiter in [None, Some("/"), Some("::")] {
2725 for versions in [false, true] {
2726 let mut model = Vec::<(String, Option<String>)>::new();
2727 let mut groups = BTreeSet::new();
2728 for (key, versions_for_key) in &history {
2729 let Some(remainder) = key.strip_prefix(prefix) else {
2730 continue;
2731 };
2732 if let Some(delimiter) = delimiter
2733 && let Some(position) = remainder.find(delimiter)
2734 {
2735 groups.insert(
2736 key.get(..prefix.len() + position + delimiter.len())
2737 .unwrap_or_else(|| unreachable!())
2738 .to_owned(),
2739 );
2740 continue;
2741 }
2742 if versions {
2743 model.extend(
2744 versions_for_key
2745 .iter()
2746 .rev()
2747 .map(|version| (key.clone(), Some(version.version_id.clone()))),
2748 );
2749 } else if let Some(version) = versions_for_key.last()
2750 && !version.delete_marker
2751 {
2752 model.push((key.clone(), Some(version.version_id.clone())));
2753 }
2754 }
2755 model.extend(groups.into_iter().map(|group| (group, None)));
2756 model.sort_by(|left, right| left.0.cmp(&right.0));
2757 for page_size in [1, 2, 5, 100] {
2758 let mut continuation = None;
2759 let mut offset = 0;
2760 loop {
2761 let page = store
2762 .list(
2763 ReadTarget::Bucket(bucket.clone()),
2764 prefix.to_owned(),
2765 delimiter.map(str::to_owned),
2766 versions,
2767 page_size,
2768 continuation,
2769 )
2770 .await
2771 .unwrap_or_else(|_| unreachable!());
2772 let expected = model
2773 .get(offset..model.len().min(offset + page_size as usize))
2774 .unwrap_or_else(|| unreachable!());
2775 let entries: Vec<_> = expected
2776 .iter()
2777 .filter_map(|(key, version)| {
2778 version.as_ref().map(|id| (key.clone(), id.clone()))
2779 })
2780 .collect();
2781 let prefixes: Vec<_> = expected
2782 .iter()
2783 .filter(|(_, version)| version.is_none())
2784 .map(|(key, _)| key.clone())
2785 .collect();
2786 assert_eq!(
2787 page.entries
2788 .iter()
2789 .map(|entry| (
2790 entry.object_key.clone(),
2791 entry
2792 .version
2793 .as_ref()
2794 .unwrap_or_else(|| unreachable!())
2795 .version_id
2796 .clone(),
2797 ))
2798 .collect::<Vec<_>>(),
2799 entries,
2800 "entries: prefix={prefix:?} delimiter={delimiter:?} versions={versions} page_size={page_size} offset={offset}"
2801 );
2802 assert_eq!(page.common_prefixes, prefixes);
2803 offset += expected.len();
2804 assert_eq!(page.continuation.is_some(), offset < model.len());
2805 let Some(token) = page.continuation else {
2806 break;
2807 };
2808 continuation = Some(token);
2809 }
2810 assert_eq!(offset, model.len());
2811 }
2812 }
2813 }
2814 }
2815 }
2816
2817 fn metadata() -> wire::ObjectMetadata {
2818 wire::ObjectMetadata {
2819 content_type: "text/plain".into(),
2820 ..Default::default()
2821 }
2822 }
2823
2824 async fn put(
2825 store: &MemoryObjects,
2826 bucket: &wire::BucketRef,
2827 object_key: &str,
2828 body: &[u8],
2829 condition: Option<Condition>,
2830 ) -> wire::ObjectVersion {
2831 store
2832 .put(PutRequest {
2833 bucket: bucket.clone(),
2834 object_key: object_key.into(),
2835 body: bytes::Bytes::copy_from_slice(body),
2836 metadata: metadata(),
2837 condition,
2838 idempotency_key: None,
2839 })
2840 .await
2841 .unwrap_or_else(|_| unreachable!())
2842 }
2843
2844 #[tokio::test]
2845 async fn synchronous_bucket_composition_is_exact_and_capacity_bounded() {
2846 assert!(matches!(
2847 MemoryObjects::new(0),
2848 Err(ObjectsError::Invalid("memory byte limit must be positive"))
2849 ));
2850 assert!(MemoryObjects::with_bucket("INVALID", 4).is_err());
2851 assert!(MemoryObjects::with_bucket_limits("embedded", 5, 4).is_err());
2852
2853 let (independent, independent_bucket) =
2854 MemoryObjects::with_bucket_limits("independent", 3, 6)
2855 .unwrap_or_else(|_| unreachable!());
2856 put(&independent, &independent_bucket, "first", b"one", None).await;
2857 put(&independent, &independent_bucket, "second", b"two", None).await;
2858 assert_eq!(
2859 independent
2860 .put(PutRequest {
2861 bucket: independent_bucket.clone(),
2862 object_key: "aggregate-overflow".into(),
2863 body: bytes::Bytes::from_static(b"x"),
2864 metadata: metadata(),
2865 condition: None,
2866 idempotency_key: None,
2867 })
2868 .await,
2869 Err(ObjectsError::Capacity)
2870 );
2871 let (per_object, per_object_bucket) = MemoryObjects::with_bucket_limits("per-object", 3, 6)
2872 .unwrap_or_else(|_| unreachable!());
2873 assert_eq!(
2874 per_object
2875 .put(PutRequest {
2876 bucket: per_object_bucket,
2877 object_key: "too-large".into(),
2878 body: bytes::Bytes::from_static(b"four"),
2879 metadata: metadata(),
2880 condition: None,
2881 idempotency_key: None,
2882 })
2883 .await,
2884 Err(ObjectsError::Capacity)
2885 );
2886
2887 let (store, bucket) =
2888 MemoryObjects::with_bucket("embedded", 4).unwrap_or_else(|_| unreachable!());
2889 assert_eq!(
2890 store
2891 .head_bucket(&bucket)
2892 .await
2893 .unwrap_or_else(|_| unreachable!())
2894 .bucket,
2895 Some(bucket.clone())
2896 );
2897 put(&store, &bucket, "full", b"four", None).await;
2898 assert_eq!(
2899 store
2900 .put(PutRequest {
2901 bucket,
2902 object_key: "overflow".into(),
2903 body: bytes::Bytes::from_static(b"x"),
2904 metadata: metadata(),
2905 condition: None,
2906 idempotency_key: None,
2907 })
2908 .await,
2909 Err(ObjectsError::Capacity)
2910 );
2911 }
2912
2913 #[tokio::test]
2914 async fn memory_provider_passes_the_public_conformance_suite() -> Result<(), ObjectsError> {
2915 crate::conformance::verify(&MemoryObjects::default(), "conformance").await
2916 }
2917
2918 #[tokio::test]
2919 async fn shared_snapshot_and_fork_bodies_retain_exact_capacity() {
2920 let store = MemoryObjects::new(3).unwrap_or_else(|_| unreachable!());
2921 let source = store
2922 .create_bucket("source-bucket".into(), Some("create-source".into()))
2923 .await
2924 .unwrap_or_else(|_| unreachable!())
2925 .bucket
2926 .unwrap_or_else(|| unreachable!());
2927 let version = put(&store, &source, "shared", b"abc", None).await;
2928 let version_id = version.version_id.clone();
2929 let snapshot = store
2930 .snapshot(source.clone(), Some("snapshot".into()))
2931 .await
2932 .unwrap_or_else(|_| unreachable!())
2933 .snapshot
2934 .unwrap_or_else(|| unreachable!());
2935 assert!(
2936 store
2937 .delete(
2938 source.clone(),
2939 "shared".into(),
2940 Some(version.version_id),
2941 None,
2942 Some("delete-source".into()),
2943 )
2944 .await
2945 .is_ok()
2946 );
2947 assert!(matches!(
2948 store
2949 .put(PutRequest {
2950 bucket: source.clone(),
2951 object_key: "replacement".into(),
2952 body: bytes::Bytes::from_static(b"def"),
2953 metadata: metadata(),
2954 condition: None,
2955 idempotency_key: None,
2956 })
2957 .await,
2958 Err(ObjectsError::Capacity)
2959 ));
2960 let fork = store
2961 .fork(
2962 ReadTarget::Snapshot(snapshot.clone()),
2963 "fork-bucket".into(),
2964 Some("fork".into()),
2965 )
2966 .await
2967 .unwrap_or_else(|_| unreachable!())
2968 .bucket
2969 .unwrap_or_else(|| unreachable!());
2970 assert!(
2971 store
2972 .destroy_snapshot(snapshot, Some("destroy-snapshot".into()))
2973 .await
2974 .unwrap_or(false)
2975 );
2976 assert!(matches!(
2977 store
2978 .put(PutRequest {
2979 bucket: source.clone(),
2980 object_key: "replacement".into(),
2981 body: bytes::Bytes::from_static(b"def"),
2982 metadata: metadata(),
2983 condition: None,
2984 idempotency_key: None,
2985 })
2986 .await,
2987 Err(ObjectsError::Capacity)
2988 ));
2989 assert!(
2990 store
2991 .delete(
2992 fork,
2993 "shared".into(),
2994 Some(version_id),
2995 None,
2996 Some("delete-fork".into()),
2997 )
2998 .await
2999 .is_ok()
3000 );
3001 assert!(
3002 store
3003 .put(PutRequest {
3004 bucket: source,
3005 object_key: "replacement".into(),
3006 body: bytes::Bytes::from_static(b"def"),
3007 metadata: metadata(),
3008 condition: None,
3009 idempotency_key: None,
3010 })
3011 .await
3012 .is_ok()
3013 );
3014 }
3015}