concat_reader/
file.rs

1use crate::ConcatRead;
2use crate::FileConcatRead;
3use std::error::Error;
4use std::fmt;
5use std::fs::File;
6use std::io::{self, Read, Result};
7use std::path::{Path, PathBuf};
8
9trait FileLike: fmt::Debug + Read + Sized {
10    fn open<P: AsRef<Path>>(p: P) -> Result<Self>;
11}
12
13impl FileLike for File {
14    #[inline]
15    fn open<P: AsRef<Path>>(p: P) -> Result<Self> {
16        File::open(p)
17    }
18}
19
20/// The `FileConcatReader` struct is a reader over multiple [`File`]'s created from an [`Iterator`] with
21/// [`AsRef<Path>`] items.
22///
23/// The reader will only attempt to open and read a file when requested.
24/// If the current reader reaches its `EOF` the `FileConcatReader` will start reading from the next
25/// path in the iterator. If all readers reached `EOF` the `FileConcatReader` will also be `EOF`.
26///
27/// # Examples
28/// ```no_run
29/// use concat_reader::*;
30/// use std::fs::File;
31/// use std::io;
32/// use std::io::prelude::*;
33///
34/// fn main() -> io::Result<()> {
35///     let files = ["foo.txt", "bar.txt", "baz.txt"];
36///     let mut c = FileConcatReader::new(&files);
37///     let mut buffer = [0; 10];
38///
39///     // read up to 10 bytes
40///     let n = c.read(&mut buffer[..])?;
41///
42///     println!("The bytes: {:?}", &buffer[..n]);
43///
44///     //skip to the next file
45///     c.skip();
46///
47///     let mut buffer = Vec::new();
48///     // read all rest files into a single buffer
49///     c.read_to_end(&mut buffer)?;
50///     Ok(())
51/// }
52/// ```
53///
54/// [`File`]:                   https://doc.rust-lang.org/std/fs/struct.File.html
55/// [`Iterator`]:               https://doc.rust-lang.org/std/iter/trait.Iterator.html
56/// [`AsRef<Path>`]:            https://doc.rust-lang.org/std/convert/trait.AsRef.html
57
58pub struct FileConcatReader<I: IntoIterator> {
59    inner: InnerReader<File, I>,
60}
61
62impl<I> FileConcatReader<I>
63where
64    I: IntoIterator,
65    I::Item: AsRef<Path>,
66{
67    /// Creates a new `FileConcatReader` from an value which can be converted
68    /// into an `Iterator<Item=AsRef<Path>>`.
69    ///
70    /// ```
71    /// use std::io::prelude::*;
72    /// use concat_reader::*;
73    /// fn main() {
74    ///     let files = ["foo.txt", "bar.txt", "baz.txt"];
75    ///     let mut c = FileConcatReader::new(&files);
76    /// }
77    /// ```
78    pub fn new(iter: I) -> Self {
79        Self {
80            inner: InnerReader::new(iter),
81        }
82    }
83}
84
85impl<I> ConcatRead for FileConcatReader<I>
86where
87    I: IntoIterator,
88    I::Item: AsRef<Path>,
89{
90    type Item = File;
91
92    fn current(&self) -> Option<&Self::Item> {
93        self.inner.current()
94    }
95
96    fn skip(&mut self) -> bool {
97        self.inner.skip()
98    }
99}
100
101impl<I> FileConcatRead for FileConcatReader<I>
102where
103    I: IntoIterator,
104    I::Item: AsRef<Path>,
105{
106    fn file_path(&self) -> Option<&Path> {
107        self.inner.file_path()
108    }
109}
110
111impl<I> From<I> for FileConcatReader<I>
112where
113    I: IntoIterator,
114    I::Item: AsRef<Path>,
115{
116    fn from(iter: I) -> Self {
117        Self::new(iter)
118    }
119}
120
121impl<I> Read for FileConcatReader<I>
122where
123    I: IntoIterator,
124    I::Item: AsRef<Path>,
125{
126    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
127        self.inner.read(buf)
128    }
129}
130
131impl<I> fmt::Debug for FileConcatReader<I>
132where
133    I: IntoIterator,
134    I::Item: fmt::Debug,
135    I::IntoIter: Clone,
136{
137    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
138        fmt::Debug::fmt(&self.inner, f)
139    }
140}
141
142enum ReaderState<R, E> {
143    Open(R, PathBuf),
144    Init(PathBuf),
145    Err(E, PathBuf),
146    Eof,
147}
148
149impl<R> ReaderState<R, io::Error>
150where
151    R: FileLike,
152{
153    fn open(&mut self) -> Result<()> {
154        use std::mem;
155        let s = match self {
156            ReaderState::Init(p) => match FileLike::open(&p) {
157                Err(e) => ReaderState::Err(e, p.clone()),
158                Ok(f) => ReaderState::Open(f, p.clone()),
159            },
160            ReaderState::Eof => panic!("called `ReaderState::open()` on a `Eof` value"),
161            ReaderState::Open(_, _) => panic!("called `ReaderState::open()` on a `Open` value"),
162            ReaderState::Err(_, _) => panic!("called `ReaderState::open()` on a `Err` value"),
163        };
164
165        mem::replace(self, s);
166        if let ReaderState::Err(e, _) = &self {
167            return Err(io::Error::new(e.kind(), e.description()));
168        }
169        Ok(())
170    }
171
172    fn is_init(&self) -> bool {
173        match *self {
174            ReaderState::Init(_) => true,
175            _ => false,
176        }
177    }
178
179    fn unwrap_err(&self) -> io::Error {
180        match self {
181            ReaderState::Err(e, _) => io::Error::new(e.kind(), e.description()),
182            _ => panic!("no error to unwrap"),
183        }
184    }
185}
186
187impl<R> Read for ReaderState<R, io::Error>
188where
189    R: FileLike,
190{
191    fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
192        match self {
193            ReaderState::Eof => Ok(0),
194            ReaderState::Init(_) => {
195                self.open()?;
196                self.read(buf)
197            }
198            ReaderState::Err(_, _) => Err(self.unwrap_err()),
199            ReaderState::Open(r, _) => r.read(buf),
200        }
201    }
202}
203
204impl<R, E, P> From<Option<P>> for ReaderState<R, E>
205where
206    P: AsRef<Path>,
207    R: FileLike,
208    E: Error,
209{
210    fn from(path: Option<P>) -> Self {
211        match path {
212            Some(p) => ReaderState::Init(p.as_ref().to_path_buf()),
213            None => ReaderState::Eof,
214        }
215    }
216}
217
218impl<R, E> fmt::Debug for ReaderState<R, E>
219where
220    R: fmt::Debug,
221    E: Error,
222{
223    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
224        match self {
225            ReaderState::Init(p) => write!(f, "ReaderState::Init({:?})", p),
226            ReaderState::Open(r, p) => write!(f, "ReaderState::Open({:?},{:?})", r, p),
227            ReaderState::Eof => write!(f, "ReaderState::Eof"),
228            ReaderState::Err(p, e) => write!(f, "ReaderState::Err({:?},{:?})", p, e),
229        }
230    }
231}
232
233struct InnerReader<R, I: IntoIterator> {
234    curr: ReaderState<R, io::Error>,
235    rest: I::IntoIter,
236}
237
238impl<R, I> InnerReader<R, I>
239where
240    R: FileLike,
241    I: IntoIterator,
242    I::Item: AsRef<Path>,
243{
244    fn new(iter: I) -> InnerReader<R, I> {
245        let mut iter = iter.into_iter();
246        let curr = iter.next().into();
247        InnerReader { curr, rest: iter }
248    }
249}
250
251impl<R, I> ConcatRead for InnerReader<R, I>
252where
253    R: FileLike,
254    I: IntoIterator,
255    I::Item: AsRef<Path>,
256{
257    type Item = R;
258
259    fn current(&self) -> Option<&Self::Item> {
260        match &self.curr {
261            ReaderState::Open(r, _) => Some(&r),
262            _ => None,
263        }
264    }
265
266    fn skip(&mut self) -> bool {
267        self.curr = self.rest.next().into();
268        self.curr.is_init()
269    }
270}
271
272impl<R, I> FileConcatRead for InnerReader<R, I>
273where
274    R: FileLike,
275    I: IntoIterator,
276    I::Item: AsRef<Path>,
277{
278    fn file_path(&self) -> Option<&Path> {
279        match &self.curr {
280            ReaderState::Init(p) => Some(p.as_path()),
281            ReaderState::Open(_, p) => Some(p.as_path()),
282            ReaderState::Err(_, p) => Some(p.as_path()),
283            _ => None,
284        }
285    }
286}
287
288impl<R, I> Read for InnerReader<R, I>
289where
290    R: FileLike,
291    I: IntoIterator,
292    I::Item: AsRef<Path>,
293{
294    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
295        if buf.is_empty() {
296            return Ok(0);
297        }
298
299        match self.curr.read(buf) {
300            Ok(0) => {
301                let has_items = self.skip();
302                if !has_items {
303                    Ok(0)
304                } else {
305                    self.read(buf)
306                }
307            }
308            val => val,
309        }
310    }
311}
312
313impl<R, I> fmt::Debug for InnerReader<R, I>
314where
315    R: fmt::Debug,
316    I: IntoIterator,
317    I::Item: fmt::Debug,
318    I::IntoIter: Clone,
319{
320    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
321        let rest: Vec<_> = self.rest.clone().collect();
322        f.debug_struct("CatReader")
323            .field("curr", &self.curr)
324            .field("rest", &rest)
325            .finish()
326    }
327}
328
329#[cfg(test)]
330mod tests {
331    use super::{FileLike, InnerReader};
332    use crate::{ConcatRead, FileConcatRead};
333    use std::io::{self, Read};
334    use std::path::Path;
335
336    impl FileLike for &'static [u8] {
337        fn open<P: AsRef<Path>>(p: P) -> io::Result<&'static [u8]> {
338            let string = p.as_ref().to_string_lossy().into_owned();
339            let reference: &str = &string;
340            match reference {
341                "test1.txt" => Ok(b"some\ntext\n"),
342                "1byte" => Ok(b"1"),
343                "2byte" => Ok(b"22"),
344                "3byte" => Ok(b"333"),
345                "4byte" => Ok(b"4444"),
346                "dir/other.test.txt" => Ok(b"here's "),
347                _ => Err(io::Error::new(io::ErrorKind::NotFound, "file missing")),
348            }
349        }
350    }
351
352    #[test]
353    fn reads_from_multiple_files() {
354        let strs = &["1byte", "2byte", "3byte"];
355        let mut reader: InnerReader<&'static [u8], _> = InnerReader::new(strs);
356
357        let mut buf = [0; 5];
358        reader.read_exact(&mut buf).unwrap();
359        assert_eq!(&buf, b"12233");
360        assert_eq!(
361            format!("{:?}", reader),
362            "CatReader { curr: ReaderState::Open([51],\"3byte\"), rest: [] }"
363        );
364    }
365
366    #[test]
367    fn init_next_reader_when_current_is_eof() {
368        let strs = &["1byte", "2byte", "3byte"];
369        let mut reader: InnerReader<&'static [u8], _> = InnerReader::new(strs);
370
371        let mut buf = [0];
372        assert_eq!(reader.read(&mut buf).unwrap(), 1);
373        assert_eq!(&buf, b"1");
374
375        assert_eq!(reader.file_path(), Some(Path::new("1byte")));
376
377        assert_eq!(reader.read(&mut buf).unwrap(), 1);
378        assert_eq!(&buf, b"2");
379
380        assert_eq!(reader.file_path(), Some(Path::new("2byte")));
381    }
382
383    #[test]
384    fn fails_on_file_error() {
385        let strs = &["1byte", "2byte", "404", "3byte", "4byte"];
386        let mut reader: InnerReader<&'static [u8], _> = InnerReader::new(strs);
387
388        let mut buf = Vec::new();
389        assert!(reader.read_to_end(&mut buf).is_err());
390
391        assert_eq!(buf, b"122");
392        assert_eq!(reader.file_path(), Some(Path::new("404")));
393    }
394
395    #[test]
396    fn can_skip_and_continue() {
397        let strs = &["404", "3byte", "4byte"];
398        let mut reader: InnerReader<&'static [u8], _> = InnerReader::new(strs);
399
400        let mut buf = Vec::new();
401        assert!(reader.read_to_end(&mut buf).is_err());
402
403        //skip to next reader
404        reader.skip();
405        assert!(reader.read_to_end(&mut buf).is_ok());
406        assert_eq!(buf, b"3334444");
407    }
408
409    #[test]
410    fn fails_on_file_error2() {
411        let strs = &["1byte", "2byte", "404", "3byte", "4byte"];
412        let mut reader: InnerReader<&'static [u8], _> = InnerReader::new(strs);
413
414        let mut buf = [0; 5];
415        assert_eq!(reader.read(&mut buf).unwrap(), 1);
416        assert_eq!(reader.read(&mut buf).unwrap(), 2);
417        assert!(reader.read(&mut buf).is_err());
418    }
419
420    #[test]
421    fn can_debug_print() {
422        let strs = &["dir/other.test.txt", "404", "test1.txt"];
423        let mut reader: InnerReader<&'static [u8], _> = InnerReader::new(strs);
424
425        assert_eq!(
426            format!("{:?}", reader),
427            "CatReader { curr: ReaderState::Init(\"dir/other.test.txt\"), rest: [\"404\", \"test1.txt\"] }"
428        );
429
430        // read zero bytes no file has been opened
431        let mut buf = [];
432        assert_eq!(reader.read(&mut buf).unwrap(), 0);
433        assert_eq!(
434            format!("{:?}", reader),
435            "CatReader { curr: ReaderState::Init(\"dir/other.test.txt\"), rest: [\"404\", \"test1.txt\"] }"
436        );
437
438        // read one byte. File should be opened
439        let mut buf = [0];
440        assert_eq!(reader.read(&mut buf).unwrap(), 1);
441        assert_eq!(buf, [104]);
442        assert_eq!(
443            format!("{:?}", reader),
444            "CatReader { curr: ReaderState::Open([101, 114, 101, 39, 115, 32],\"dir/other.test.txt\"), rest: [\"404\", \"test1.txt\"] }"
445        );
446
447        // read rest of files and fail because of missing file
448        let mut buf = Vec::new();
449        assert!(reader.read_to_end(&mut buf).is_err());
450        assert_eq!(
451            format!("{:?}", reader),
452            "CatReader { curr: ReaderState::Err(Custom { kind: NotFound, error: \"file missing\" },\"404\"), rest: [\"test1.txt\"] }"
453        );
454
455        assert!(reader.read_to_end(&mut buf).is_err());
456        assert_eq!(
457            format!("{:?}", reader),
458            "CatReader { curr: ReaderState::Err(Custom { kind: NotFound, error: \"file missing\" },\"404\"), rest: [\"test1.txt\"] }"
459        );
460        // we can skip the file if we want
461        reader.skip();
462        assert_eq!(
463            format!("{:?}", reader),
464            "CatReader { curr: ReaderState::Init(\"test1.txt\"), rest: [] }"
465        );
466
467        assert_eq!(reader.read_to_end(&mut buf).unwrap(), 10);
468        assert_eq!(buf, b"ere's some\ntext\n");
469        assert_eq!(
470            format!("{:?}", reader),
471            "CatReader { curr: ReaderState::Eof, rest: [] }"
472        );
473    }
474}
concat_reader/file.rs

concat_reader/
file.rs
