1use std::path::{Path, PathBuf};
18use std::sync::Arc;
19use std::time::Duration;
20
21use async_trait::async_trait;
22use bytes::Bytes;
23use futures_core::Stream;
24use tokio::io::AsyncWriteExt as _;
25use tokio_util::io::ReaderStream;
26
27use cratefield_core::{
28 Blob, BlobError, BlobMeta, BlobObject, BlobPage, BlobStream, BoxStream, MAX_LIST_LIMIT,
29 MIN_MULTIPART_PART_BYTES, PartReceipt, PendingUpload, StreamError, UploadId,
30 blob_error_from_stream, check_blob_size, check_part_number, limit_stream,
31};
32
33const TMP_DIR: &str = ".__tmp";
36
37const MULTIPART_DIR: &str = ".__multipart";
39
40const MULTIPART_KEY_FILE: &str = "key";
42
43const MULTIPART_CONTENT_TYPE_FILE: &str = "content_type";
45
46const CONTENT_TYPE_SUFFIX: &str = ".__ct";
48
49const READ_CHUNK: usize = 64 * 1024;
51
52const DEFAULT_CONTENT_TYPE: &str = "application/octet-stream";
55
56fn part_file(part_number: u16) -> String {
59 format!("part-{part_number:05}")
60}
61
62fn part_etag(part_number: u16, len: u64) -> String {
66 format!("part-{part_number}-{len}")
67}
68
69fn unique_token() -> String {
75 use std::sync::atomic::{AtomicU64, Ordering};
76 use std::time::{SystemTime, UNIX_EPOCH};
77
78 static COUNTER: AtomicU64 = AtomicU64::new(0);
79 let seq = COUNTER.fetch_add(1, Ordering::Relaxed);
80 let nanos = SystemTime::now()
81 .duration_since(UNIX_EPOCH)
82 .map_or(0, |elapsed| elapsed.as_nanos());
83 format!("{nanos:x}-{:x}-{seq:x}", std::process::id())
84}
85
86async fn validate_parts(dir: &Path, ordered: &[PartReceipt]) -> Result<(), BlobError> {
92 for (index, receipt) in ordered.iter().enumerate() {
93 let path = dir.join(part_file(receipt.part_number));
94 let meta = match tokio::fs::metadata(&path).await {
95 Ok(meta) => meta,
96 Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
97 return Err(BlobError::Operation(format!(
98 "part {} was never uploaded",
99 receipt.part_number
100 )));
101 }
102 Err(err) => return Err(DirBlob::io_err(&err)),
103 };
104 if part_etag(receipt.part_number, meta.len()) != receipt.etag {
105 return Err(BlobError::Operation(format!(
106 "part {} etag mismatch",
107 receipt.part_number
108 )));
109 }
110 let is_last = index + 1 == ordered.len();
111 if !is_last && meta.len() < MIN_MULTIPART_PART_BYTES {
112 return Err(BlobError::Operation(format!(
113 "part {} is below the {MIN_MULTIPART_PART_BYTES}-byte minimum for a non-final part",
114 receipt.part_number,
115 )));
116 }
117 }
118 Ok(())
119}
120
121pub struct DirBlob {
123 base: PathBuf,
124}
125
126impl DirBlob {
127 #[must_use]
129 pub fn new(base: impl Into<PathBuf>) -> Self {
130 Self { base: base.into() }
131 }
132
133 #[must_use]
135 pub fn arc(base: impl Into<PathBuf>) -> Arc<dyn Blob> {
136 Arc::new(Self::new(base))
137 }
138
139 fn object_path(&self, key: &str) -> Result<PathBuf, BlobError> {
143 if key.is_empty() {
144 return Err(BlobError::BadKey("a blob key cannot be empty".to_owned()));
145 }
146 if key.starts_with('/') {
147 return Err(BlobError::BadKey(format!("key `{key}` must be relative")));
148 }
149 if key
150 .split('/')
151 .any(|segment| segment == ".." || segment == ".")
152 {
153 return Err(BlobError::BadKey(format!(
154 "key `{key}` must not contain `.` or `..` segments"
155 )));
156 }
157 Ok(self.base.join(key))
158 }
159
160 fn content_type_path(path: &Path) -> PathBuf {
161 PathBuf::from(format!("{}{CONTENT_TYPE_SUFFIX}", path.display()))
162 }
163
164 async fn read_content_type(path: &Path) -> String {
167 tokio::fs::read_to_string(Self::content_type_path(path))
168 .await
169 .unwrap_or_else(|_| DEFAULT_CONTENT_TYPE.to_owned())
170 }
171
172 fn io_err(err: &std::io::Error) -> BlobError {
173 BlobError::Operation(err.to_string())
174 }
175
176 fn tmp_dir(&self) -> PathBuf {
178 self.base.join(TMP_DIR)
179 }
180
181 fn upload_dir(&self, upload_id: &UploadId) -> Result<PathBuf, BlobError> {
186 let raw = upload_id.as_str();
187 if raw.is_empty() || raw == "." || raw == ".." || raw.contains('/') || raw.contains('\\') {
188 return Err(BlobError::Operation(format!(
189 "invalid multipart upload id `{raw}`"
190 )));
191 }
192 Ok(self.base.join(MULTIPART_DIR).join(raw))
193 }
194
195 async fn read_upload_key(dir: &Path, upload_id: &UploadId) -> Result<String, BlobError> {
197 match tokio::fs::read_to_string(dir.join(MULTIPART_KEY_FILE)).await {
198 Ok(key) => Ok(key),
199 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Err(BlobError::Operation(
200 format!("no multipart upload `{}`", upload_id.as_str()),
201 )),
202 Err(err) => Err(Self::io_err(&err)),
203 }
204 }
205
206 async fn stream_to_file(
211 path: &Path,
212 body: BoxStream<'static, Result<Bytes, StreamError>>,
213 max_bytes: u64,
214 ) -> Result<u64, BlobError> {
215 let mut file = tokio::fs::File::create(path)
216 .await
217 .map_err(|err| Self::io_err(&err))?;
218 let mut limited = limit_stream(body, max_bytes);
219 let result: Result<u64, BlobError> = async {
220 let mut written: u64 = 0;
221 while let Some(chunk) = std::future::poll_fn(|cx| limited.as_mut().poll_next(cx)).await
222 {
223 let chunk = chunk.map_err(blob_error_from_stream)?;
224 file.write_all(&chunk)
225 .await
226 .map_err(|err| Self::io_err(&err))?;
227 written = written.saturating_add(chunk.len() as u64);
228 }
229 file.flush().await.map_err(|err| Self::io_err(&err))?;
230 Ok(written)
231 }
232 .await;
233 if result.is_err() {
234 let _ = tokio::fs::remove_file(path).await;
235 }
236 result
237 }
238
239 async fn place_object(tmp: &Path, path: &Path, content_type: &str) -> Result<(), BlobError> {
243 if let Some(parent) = path.parent() {
244 tokio::fs::create_dir_all(parent)
245 .await
246 .map_err(|err| Self::io_err(&err))?;
247 }
248 tokio::fs::write(Self::content_type_path(path), content_type.as_bytes())
249 .await
250 .map_err(|err| Self::io_err(&err))?;
251 tokio::fs::rename(tmp, path)
252 .await
253 .map_err(|err| Self::io_err(&err))
254 }
255
256 async fn meta_for(&self, key: &str) -> Result<Option<BlobMeta>, BlobError> {
259 let path = self.object_path(key)?;
260 let meta = match tokio::fs::metadata(&path).await {
261 Ok(meta) => meta,
262 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
263 Err(err) => return Err(Self::io_err(&err)),
264 };
265 if !meta.is_file() {
266 return Ok(None);
267 }
268 Ok(Some(BlobMeta {
269 key: key.to_owned(),
270 size: meta.len(),
271 content_type: Self::read_content_type(&path).await,
272 }))
273 }
274
275 async fn collect_keys(&self, prefix: &str) -> Result<Vec<String>, BlobError> {
280 let mut keys = Vec::new();
281 let mut stack = vec![self.base.clone()];
282 while let Some(dir) = stack.pop() {
283 let mut entries = match tokio::fs::read_dir(&dir).await {
284 Ok(entries) => entries,
285 Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
286 Err(err) => return Err(Self::io_err(&err)),
287 };
288 while let Some(entry) = entries
289 .next_entry()
290 .await
291 .map_err(|err| Self::io_err(&err))?
292 {
293 let file_type = entry.file_type().await.map_err(|err| Self::io_err(&err))?;
294 let name = entry.file_name();
295 let name = name.to_string_lossy();
296 if file_type.is_dir() {
297 if name == TMP_DIR || name == MULTIPART_DIR {
298 continue;
299 }
300 stack.push(entry.path());
301 } else if file_type.is_file() {
302 if name.ends_with(CONTENT_TYPE_SUFFIX) {
303 continue;
304 }
305 if let Ok(rel) = entry.path().strip_prefix(&self.base) {
306 let key = rel
307 .components()
308 .map(|part| part.as_os_str().to_string_lossy().into_owned())
309 .collect::<Vec<_>>()
310 .join("/");
311 if key.starts_with(prefix) {
312 keys.push(key);
313 }
314 }
315 }
316 }
317 }
318 keys.sort();
319 Ok(keys)
320 }
321
322 async fn read_upload_content_type(dir: &Path) -> String {
324 tokio::fs::read_to_string(dir.join(MULTIPART_CONTENT_TYPE_FILE))
325 .await
326 .unwrap_or_else(|_| DEFAULT_CONTENT_TYPE.to_owned())
327 }
328}
329
330#[async_trait]
331impl Blob for DirBlob {
332 async fn put(&self, key: &str, bytes: &[u8], content_type: &str) -> Result<(), BlobError> {
333 check_blob_size(bytes)?;
334 let path = self.object_path(key)?;
335 if let Some(parent) = path.parent() {
336 tokio::fs::create_dir_all(parent)
337 .await
338 .map_err(|err| Self::io_err(&err))?;
339 }
340 tokio::fs::write(&path, bytes)
341 .await
342 .map_err(|err| Self::io_err(&err))?;
343 tokio::fs::write(Self::content_type_path(&path), content_type.as_bytes())
344 .await
345 .map_err(|err| Self::io_err(&err))?;
346 Ok(())
347 }
348
349 async fn get(&self, key: &str) -> Result<Option<BlobObject>, BlobError> {
350 let path = self.object_path(key)?;
351 let bytes = match tokio::fs::read(&path).await {
352 Ok(bytes) => bytes,
353 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
354 Err(err) => return Err(Self::io_err(&err)),
355 };
356 Ok(Some(BlobObject {
357 bytes,
358 content_type: Self::read_content_type(&path).await,
359 }))
360 }
361
362 async fn delete(&self, key: &str) -> Result<(), BlobError> {
363 let path = self.object_path(key)?;
364 for target in [path.clone(), Self::content_type_path(&path)] {
365 match tokio::fs::remove_file(&target).await {
366 Ok(()) => {}
367 Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
368 Err(err) => return Err(Self::io_err(&err)),
369 }
370 }
371 Ok(())
372 }
373
374 async fn signed_url(&self, _key: &str, _ttl: Duration) -> Result<String, BlobError> {
377 Err(BlobError::Unsupported(
378 "a directory store has no presigned URLs; serve the bytes through the harness \
379 or use an S3-compatible adapter"
380 .to_owned(),
381 ))
382 }
383
384 async fn put_stream(
385 &self,
386 key: &str,
387 body: BoxStream<'static, Result<Bytes, StreamError>>,
388 content_type: &str,
389 max_bytes: u64,
390 ) -> Result<u64, BlobError> {
391 let path = self.object_path(key)?;
392 let tmp_dir = self.tmp_dir();
393 tokio::fs::create_dir_all(&tmp_dir)
394 .await
395 .map_err(|err| Self::io_err(&err))?;
396 let tmp = tmp_dir.join(unique_token());
397 let written = Self::stream_to_file(&tmp, body, max_bytes).await?;
398 if let Err(err) = Self::place_object(&tmp, &path, content_type).await {
399 let _ = tokio::fs::remove_file(&tmp).await;
400 return Err(err);
401 }
402 Ok(written)
403 }
404
405 async fn get_stream(&self, key: &str) -> Result<Option<BlobStream>, BlobError> {
406 let path = self.object_path(key)?;
407 let file = match tokio::fs::File::open(&path).await {
408 Ok(file) => file,
409 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
410 Err(err) => return Err(Self::io_err(&err)),
411 };
412 let size = file
413 .metadata()
414 .await
415 .map_err(|err| Self::io_err(&err))?
416 .len();
417 let content_type = Self::read_content_type(&path).await;
418 Ok(Some(BlobStream {
419 content_type,
420 size,
421 body: Box::pin(MapIoError {
422 inner: ReaderStream::with_capacity(file, READ_CHUNK),
423 }),
424 }))
425 }
426
427 async fn head(&self, key: &str) -> Result<Option<BlobMeta>, BlobError> {
428 self.meta_for(key).await
429 }
430
431 async fn list(
432 &self,
433 prefix: &str,
434 cursor: Option<&str>,
435 limit: usize,
436 ) -> Result<BlobPage, BlobError> {
437 let limit = limit.clamp(1, MAX_LIST_LIMIT);
438 let keys = self.collect_keys(prefix).await?;
439 let start = cursor.map_or(0, |cursor| {
440 keys.partition_point(|key| key.as_str() <= cursor)
441 });
442 let take = (keys.len() - start).min(limit);
443 let selected = &keys[start..start + take];
444 let mut objects = Vec::with_capacity(selected.len());
445 for key in selected {
446 if let Some(meta) = self.meta_for(key).await? {
447 objects.push(meta);
448 }
449 }
450 let cursor = if start + take < keys.len() {
451 selected.last().cloned()
452 } else {
453 None
454 };
455 Ok(BlobPage { objects, cursor })
456 }
457
458 async fn create_multipart(&self, key: &str, content_type: &str) -> Result<UploadId, BlobError> {
459 let _ = self.object_path(key)?;
462 let upload_id = UploadId::new(unique_token());
463 let dir = self.upload_dir(&upload_id)?;
464 tokio::fs::create_dir_all(&dir)
465 .await
466 .map_err(|err| Self::io_err(&err))?;
467 tokio::fs::write(dir.join(MULTIPART_KEY_FILE), key.as_bytes())
468 .await
469 .map_err(|err| Self::io_err(&err))?;
470 tokio::fs::write(
471 dir.join(MULTIPART_CONTENT_TYPE_FILE),
472 content_type.as_bytes(),
473 )
474 .await
475 .map_err(|err| Self::io_err(&err))?;
476 Ok(upload_id)
477 }
478
479 async fn upload_part(
480 &self,
481 key: &str,
482 upload_id: &UploadId,
483 part_number: u16,
484 body: BoxStream<'static, Result<Bytes, StreamError>>,
485 max_part_bytes: u64,
486 ) -> Result<PartReceipt, BlobError> {
487 check_part_number(part_number)?;
488 let dir = self.upload_dir(upload_id)?;
489 let stored = Self::read_upload_key(&dir, upload_id).await?;
490 if stored != key {
491 return Err(BlobError::Operation(
492 "multipart upload belongs to a different key".to_owned(),
493 ));
494 }
495 let tmp_dir = self.tmp_dir();
496 tokio::fs::create_dir_all(&tmp_dir)
497 .await
498 .map_err(|err| Self::io_err(&err))?;
499 let tmp = tmp_dir.join(unique_token());
500 let written = Self::stream_to_file(&tmp, body, max_part_bytes).await?;
501 tokio::fs::rename(&tmp, dir.join(part_file(part_number)))
502 .await
503 .map_err(|err| {
504 let _ = std::fs::remove_file(&tmp);
507 Self::io_err(&err)
508 })?;
509 Ok(PartReceipt {
510 part_number,
511 etag: part_etag(part_number, written),
512 })
513 }
514
515 async fn complete_multipart(
516 &self,
517 key: &str,
518 upload_id: &UploadId,
519 parts: &[PartReceipt],
520 ) -> Result<(), BlobError> {
521 let dir = self.upload_dir(upload_id)?;
522 let stored = Self::read_upload_key(&dir, upload_id).await?;
523 if stored != key {
524 return Err(BlobError::Operation(
525 "multipart upload belongs to a different key".to_owned(),
526 ));
527 }
528 let mut ordered = parts.to_vec();
529 ordered.sort_by_key(|receipt| receipt.part_number);
530 validate_parts(&dir, &ordered).await?;
531
532 let content_type = Self::read_upload_content_type(&dir).await;
533 let path = self.object_path(key)?;
534 let tmp_dir = self.tmp_dir();
535 tokio::fs::create_dir_all(&tmp_dir)
536 .await
537 .map_err(|err| Self::io_err(&err))?;
538 let tmp = tmp_dir.join(unique_token());
539 let assembled = async {
540 let mut dst = tokio::fs::File::create(&tmp)
541 .await
542 .map_err(|err| Self::io_err(&err))?;
543 for receipt in &ordered {
544 let mut src = tokio::fs::File::open(dir.join(part_file(receipt.part_number)))
545 .await
546 .map_err(|err| Self::io_err(&err))?;
547 tokio::io::copy(&mut src, &mut dst)
548 .await
549 .map_err(|err| Self::io_err(&err))?;
550 }
551 dst.flush().await.map_err(|err| Self::io_err(&err))
552 }
553 .await;
554 if let Err(err) = assembled {
555 let _ = tokio::fs::remove_file(&tmp).await;
556 return Err(err);
557 }
558 if let Err(err) = Self::place_object(&tmp, &path, &content_type).await {
559 let _ = tokio::fs::remove_file(&tmp).await;
560 return Err(err);
561 }
562 let _ = tokio::fs::remove_dir_all(&dir).await;
563 Ok(())
564 }
565
566 async fn abort_multipart(&self, key: &str, upload_id: &UploadId) -> Result<(), BlobError> {
567 let dir = self.upload_dir(upload_id)?;
568 let stored = Self::read_upload_key(&dir, upload_id).await?;
569 if stored != key {
570 return Err(BlobError::Operation(
571 "multipart upload belongs to a different key".to_owned(),
572 ));
573 }
574 tokio::fs::remove_dir_all(&dir)
575 .await
576 .map_err(|err| Self::io_err(&err))
577 }
578
579 async fn list_multipart_uploads(&self, prefix: &str) -> Result<Vec<PendingUpload>, BlobError> {
580 let root = self.base.join(MULTIPART_DIR);
581 let mut entries = match tokio::fs::read_dir(&root).await {
582 Ok(entries) => entries,
583 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
584 Err(err) => return Err(Self::io_err(&err)),
585 };
586 let mut pending = Vec::new();
587 while let Some(entry) = entries
588 .next_entry()
589 .await
590 .map_err(|err| Self::io_err(&err))?
591 {
592 let Ok(key) = tokio::fs::read_to_string(entry.path().join(MULTIPART_KEY_FILE)).await
593 else {
594 continue;
595 };
596 if key.starts_with(prefix) {
597 let upload_id = UploadId::new(entry.file_name().to_string_lossy().into_owned());
598 pending.push(PendingUpload { key, upload_id });
599 }
600 }
601 pending.sort_by(|a, b| a.key.cmp(&b.key));
602 Ok(pending)
603 }
604}
605
606struct MapIoError<S> {
610 inner: S,
611}
612
613impl<S> Stream for MapIoError<S>
614where
615 S: Stream<Item = Result<Bytes, std::io::Error>> + Unpin,
616{
617 type Item = Result<Bytes, StreamError>;
618
619 fn poll_next(
620 self: std::pin::Pin<&mut Self>,
621 cx: &mut std::task::Context<'_>,
622 ) -> std::task::Poll<Option<Self::Item>> {
623 let inner = std::pin::Pin::new(&mut self.get_mut().inner);
624 inner.poll_next(cx).map(|polled| {
625 polled.map(|item| item.map_err(|err| StreamError::Transport(err.to_string())))
626 })
627 }
628}
629
630#[cfg(test)]
631mod tests {
632 use super::*;
633 use std::pin::Pin;
634 use std::task::{Context, Poll};
635
636 use cratefield_testing::assert_blob_large_round_trips;
637
638 fn temp_dir() -> PathBuf {
639 std::env::temp_dir().join(format!("cf-blob-{}", unique_token()))
640 }
641
642 struct Chunks(std::vec::IntoIter<Result<Bytes, StreamError>>);
644
645 impl Stream for Chunks {
646 type Item = Result<Bytes, StreamError>;
647
648 fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
649 Poll::Ready(self.get_mut().0.next())
650 }
651 }
652
653 fn chunk_stream(chunks: Vec<Vec<u8>>) -> BoxStream<'static, Result<Bytes, StreamError>> {
655 Box::pin(Chunks(
656 chunks
657 .into_iter()
658 .map(|chunk| Ok(Bytes::from(chunk)))
659 .collect::<Vec<_>>()
660 .into_iter(),
661 ))
662 }
663
664 #[tokio::test]
665 async fn round_trips_bytes_and_content_type() {
666 let base = temp_dir();
667 let blob = DirBlob::new(&base);
668 blob.put("cms/hero.png", b"\x89PNG", "image/png")
669 .await
670 .unwrap();
671 let got = blob.get("cms/hero.png").await.unwrap().expect("present");
672 assert_eq!(got.bytes, b"\x89PNG");
673 assert_eq!(got.content_type, "image/png");
674 assert!(blob.get("cms/missing.png").await.unwrap().is_none());
675 let _ = std::fs::remove_dir_all(&base);
676 }
677
678 #[tokio::test]
679 async fn delete_is_idempotent_and_removes_both_files() {
680 let base = temp_dir();
681 let blob = DirBlob::new(&base);
682 blob.put("a/b.txt", b"hi", "text/plain").await.unwrap();
683 blob.delete("a/b.txt").await.unwrap();
684 assert!(blob.get("a/b.txt").await.unwrap().is_none());
685 blob.delete("a/b.txt").await.unwrap();
687 let _ = std::fs::remove_dir_all(&base);
688 }
689
690 #[tokio::test]
691 async fn an_escaping_key_is_refused() {
692 let blob = DirBlob::new(temp_dir());
693 for bad in ["", "/etc/passwd", "../x", "a/../../b", "."] {
694 assert!(matches!(
695 blob.get(bad).await.unwrap_err(),
696 BlobError::BadKey(_)
697 ));
698 }
699 }
700
701 #[tokio::test]
705 async fn dir_blob_satisfies_the_large_blob_contract() {
706 let base = temp_dir();
707 assert_blob_large_round_trips(DirBlob::arc(&base)).await;
708 let _ = std::fs::remove_dir_all(&base);
709 }
710
711 #[tokio::test]
714 async fn a_refused_streamed_put_leaves_no_temp_files() {
715 let base = temp_dir();
716 let blob = DirBlob::new(&base);
717 let err = blob
718 .put_stream(
719 "cms/big.bin",
720 chunk_stream(vec![vec![0_u8; 4096]]),
721 "application/octet-stream",
722 1024,
723 )
724 .await
725 .expect_err("4096 bytes over a 1024-byte ceiling is refused");
726 assert!(matches!(err, BlobError::TooLarge(_)), "got {err:?}");
727 assert!(blob.get("cms/big.bin").await.unwrap().is_none());
728
729 let mut leftover = 0;
730 if let Ok(mut entries) = tokio::fs::read_dir(base.join(TMP_DIR)).await {
731 while entries.next_entry().await.unwrap().is_some() {
732 leftover += 1;
733 }
734 }
735 assert_eq!(leftover, 0, "no staging file remains after the refusal");
736 let _ = std::fs::remove_dir_all(&base);
737 }
738}