chunky-bits 0.2.0

Chunky Bits is simple, unmanaged, distributed HTTP object store tool.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

use std::borrow::Borrow;

use futures::stream::{
    self,
    Stream,
    StreamExt,
};
use tokio::io::{
    self,
    AsyncRead,
};
use tokio_util::io::StreamReader;

use crate::{
    error::FileReadError,
    file::{
        FilePart,
        FileReference,
        LocationContext,
    },
};

pub struct FileReadBuilder<T> {
    file: T,
    buffer: usize,
    location_context: LocationContext,
}

impl<T> FileReadBuilder<T> {
    pub(super) fn new(file: T) -> FileReadBuilder<T> {
        FileReadBuilder {
            file,
            buffer: 5,
            location_context: Default::default(),
        }
    }

    pub fn location_context(self, location_context: LocationContext) -> FileReadBuilder<T> {
        let mut new = self;
        new.location_context = location_context;
        new
    }

    fn inner_stream_reader<'a>(
        &self,
        parts: impl Iterator<Item = impl Borrow<FilePart> + Send + Sync> + Send + 'a,
        length: u64,
    ) -> impl Stream<Item = Result<Vec<u8>, FileReadError>> + Send + 'a {
        let FileReadBuilder {
            buffer,
            location_context,
            ..
        } = self;
        let mut bytes_remaining: u64 = length;
        let location_context = location_context.clone();
        stream::iter(parts.map(move |part| {
            let location_context = location_context.clone();
            async move { part.borrow().read_with_context(&location_context).await }
        }))
        .buffered(*buffer)
        .map(move |res| match res {
            Ok(mut bytes) => {
                if bytes.len() as u64 > bytes_remaining {
                    bytes.drain((bytes_remaining as usize)..);
                }
                bytes_remaining -= bytes.len() as u64;
                Ok(bytes)
            },
            Err(err) => Err(err),
        })
    }

    fn inner_reader<'a>(
        stream: impl Stream<Item = Result<Vec<u8>, FileReadError>> + Send + 'a,
    ) -> impl AsyncRead + Send + 'a {
        StreamReader::new(stream.map(|res| -> io::Result<_> {
            match res {
                Ok(bytes) => Ok(std::io::Cursor::new(bytes)),
                Err(err) => Err(io::Error::new(io::ErrorKind::Other, err)),
            }
        }))
    }
}

impl<T> FileReadBuilder<T>
where
    T: Borrow<FileReference> + Send + Sync,
{
    pub fn stream_reader(&self) -> impl Stream<Item = Result<Vec<u8>, FileReadError>> + Send + '_ {
        let FileReference { parts, length, .. } = self.file.borrow();
        self.inner_stream_reader(parts.iter(), length.unwrap())
    }

    pub fn reader(&self) -> impl AsyncRead + Send + Unpin + '_ {
        Self::inner_reader(self.stream_reader())
    }
}

impl<T> FileReadBuilder<T>
where
    T: Into<FileReference> + Send + 'static,
{
    pub fn stream_reader_owned(
        self,
    ) -> impl Stream<Item = Result<Vec<u8>, FileReadError>> + Send + 'static {
        let FileReadBuilder {
            buffer,
            location_context,
            file,
        } = self;
        let new = FileReadBuilder {
            buffer,
            location_context,
            file: (),
        };
        let file = file.into();
        let length = file.length.unwrap();
        let parts = file.parts.into_iter();
        new.inner_stream_reader(parts, length)
    }

    pub fn reader_owned(self) -> impl AsyncRead + Send + Unpin + 'static {
        FileReadBuilder::<()>::inner_reader(self.stream_reader_owned())
    }
}