keeshond_datapack/
source.rs

1use std::collections::HashMap;
2use std::fs::File;
3use std::hash::BuildHasherDefault;
4use std::io::{BufReader, Write, BufWriter, Seek, SeekFrom, Read};
5use std::iter::Iterator;
6use std::path::{PathBuf, StripPrefixError};
7
8use crate::ReadSeek;
9
10use walkdir::{IntoIter, WalkDir};
11use failure::Fail;
12use rustc_hash::FxHasher;
13
14use zip::ZipArchive;
15use zip::result::ZipError;
16
17/// A numeric ID used to refer to a [Source].
18pub type SourceId = usize;
19
20/// Trust settings for a given [Source]
21#[derive(Debug, Copy, Clone, Eq, PartialEq)]
22pub enum TrustLevel
23{
24    /// The source is untrusted (it cannot load resources that require trust)
25    UntrustedSource,
26    /// The source is trusted (it can load resources that require trust)
27    TrustedSource
28}
29
30/// Holds a list of [Source] objects and selects one to use when loading a package
31pub struct SourceManager
32{
33    sources : Vec<Box<dyn Source>>
34}
35
36impl SourceManager
37{
38    /// Constructs a new, empty [SourceManager]
39    pub fn new() -> SourceManager
40    {
41        SourceManager
42        {
43            sources : Vec::new()
44        }
45    }
46
47    /// Adds the given [Source] to the end of the list.
48    pub fn add_source(&mut self, source : Box<dyn Source>) -> SourceId
49    {
50        self.sources.push(source);
51
52        self.sources.len()
53    }
54
55    /// Returns a reference to the [Source] of the given ID.
56    pub fn source(&mut self, id : SourceId) -> Option<&mut Box<dyn Source>>
57    {
58        if id == 0 || id - 1 >= self.sources.len()
59        {
60            return None
61        }
62
63        Some(&mut self.sources[id - 1])
64    }
65
66    /// Returns the [SourceId] of the [Source] that has the given package, going in
67    ///  reverse order of when they were added. If no suitable [Source] is available,
68    ///  returns a "null" ID.
69    pub fn package_source_id(&self, package_name : &str) -> SourceId
70    {
71        for i in (0..self.sources.len()).rev()
72        {
73            if self.sources[i].has_package(package_name)
74            {
75                return i + 1;
76            }
77        }
78
79        0
80    }
81
82    /// Retrieves a mutable reference to a [Source] that has the given package, going in
83    ///  reverse order of when they were added. If no suitable [Source] is available,
84    ///  returns [None].
85    pub fn package_source(&mut self, package_name : &str) -> Option<&mut Box<dyn Source>>
86    {
87        for source in self.sources.iter_mut().rev()
88        {
89            if source.has_package(package_name)
90            {
91                return Some(source);
92            }
93        }
94
95        None
96    }
97
98    /// Removes all sources from the manager. Existing source IDs are invalidated.
99    pub fn clear(&mut self)
100    {
101        self.sources.clear();
102    }
103}
104
105/// An error returned when failing to access a package in a [Source]
106#[derive(Debug, Fail)]
107pub enum PackageError
108{
109    /// An error occurred while iterating through the packages, but no more information is available.
110    ///  You should generally avoid using this unless you have to.
111    #[fail(display = "Failed to access package source")]
112    Generic,
113    /// An error of type [std::io::Error] occurred.
114    #[fail(display = "{}", _0)]
115    IoError(#[cause] std::io::Error),
116    /// A package item's path does not belong to the package's path.
117    #[fail(display = "Package item does not belong to the given prefix: {}", _0)]
118    PrefixMismatch(StripPrefixError),
119    /// The package has no [crate::DataObject] by the given pathname
120    #[fail(display = "The data object was not found")]
121    DataNotFound,
122    /// The operation is not supported by this implementation
123    #[fail(display = "Operation not supported")]
124    NotSupported,
125    /// A logical error occurred while reading a source
126    #[fail(display = "The package contained invalid data: {}", _0)]
127    BadData(String)
128}
129
130impl From<std::io::Error> for PackageError
131{
132    fn from(error : std::io::Error) -> PackageError
133    {
134        PackageError::IoError(error)
135    }
136}
137
138/// Represents a location that packages can be loaded from. For example, you could load packages from the
139///  filesystem (via [FilesystemSource], or out of a special archive format.
140pub trait Source
141{
142    /// The path that this [Source] originates from. Only used for debug purposes.
143    fn get_uri(&self) -> &str;
144    /// Returns true if the [Source] has a package of the given name, otherwise returns false
145    fn has_package(&self, package_name : &str) -> bool;
146    /// Returns a list of all packages available in this [Source]. Do not call this repeatedly!
147    fn list_packages(&mut self) -> Vec<String>;
148    /// Returns a [Read] + [Seek] for the file at the given pathname, if one exists.
149    fn read_file<'a>(&'a mut self, package_name: &str, pathname: &str) -> Result<Box<dyn ReadSeek + 'a>, PackageError>;
150    /// Returns a [Write] for the file at the given pathname.
151    #[allow(unused_variables)]
152    fn write_file<'a>(&'a mut self, package_name : &str, pathname : &str) -> Result<Box<dyn Write + 'a>, PackageError> { Err(PackageError::NotSupported) }
153    /// Returns an iterator through the items in a given package, if the [Source] has said package
154    fn iter_entries<'a>(&'a mut self, package_name : &str, type_folder : &str) -> Box<dyn Iterator<Item = Result<String, PackageError>> + 'a>;
155    /// Returns the source's trust level for the given package. Trusted sources are able to load
156    ///  resource types marked as requiring trust.
157    fn trust_level(&self, package_name : &str) -> TrustLevel;
158}
159
160
161////////////////////////////////////////////////////////////////////////////////
162
163
164struct EmptyEntryIter {}
165
166impl EmptyEntryIter
167{
168    fn new() -> EmptyEntryIter { EmptyEntryIter {} }
169}
170
171impl Iterator for EmptyEntryIter
172{
173    type Item = Result<String, PackageError>;
174
175    fn next(&mut self) -> Option<Result<String, PackageError>>
176    {
177        None
178    }
179}
180
181
182struct FilesystemIter
183{
184    basepath : PathBuf,
185    walkdir : IntoIter
186}
187
188impl FilesystemIter
189{
190    fn new<P : Into<PathBuf>>(basepath : P) -> FilesystemIter
191    {
192        let path = basepath.into();
193
194        FilesystemIter
195        {
196            basepath : path.clone(),
197            walkdir : WalkDir::new(path).into_iter(),
198        }
199    }
200}
201
202impl Iterator for FilesystemIter
203{
204    type Item = Result<String, PackageError>;
205
206    fn next(&mut self) -> Option<Result<String, PackageError>>
207    {
208        while let Some(dir_next) = self.walkdir.next()
209        {
210            let dir_entry = match dir_next
211            {
212                Ok(dir_entry) => { dir_entry }
213                Err(error) =>
214                {
215                    if let Some(io_error) = error.io_error()
216                    {
217                        if io_error.kind() == std::io::ErrorKind::NotFound
218                        {
219                            return None;
220                        }
221                    }
222
223                    return Some(Err(PackageError::IoError(error.into())));
224                }
225            };
226
227            if !dir_entry.file_type().is_file()
228            {
229                continue;
230            }
231
232            let path = dir_entry.path();
233            let filepath = match path.strip_prefix(self.basepath.clone())
234            {
235                Ok(filepath) => { filepath }
236                Err(error) =>
237                {
238                    return Some(Err(PackageError::PrefixMismatch(error)));
239                }
240            };
241
242            let mut pathname = String::new();
243            let mut first_part = true;
244
245            for part in filepath.iter()
246            {
247                if !first_part
248                {
249                    pathname += "/";
250                }
251                pathname += &part.to_string_lossy();
252
253                first_part = false;
254            }
255
256            return Some(Ok(pathname));
257        }
258
259        None
260    }
261}
262
263/// A [Source] that loads packages from the filesystem. This is a good source to use for
264///  development, or if you don't care about packaging your data files into an archive.
265pub struct FilesystemSource
266{
267    basedir : String,
268    package_list : Option<Vec<String>>,
269    trust : TrustLevel
270}
271
272impl FilesystemSource
273{
274    /// Creates a new [FilesystemSource] using the given directory to look for packages in
275    pub fn new(directory : &str, trust : TrustLevel) -> FilesystemSource
276    {
277        FilesystemSource
278        {
279            basedir : directory.to_string(),
280            package_list : None,
281            trust
282        }
283    }
284}
285
286impl Source for FilesystemSource
287{
288    fn get_uri(&self) -> &str
289    {
290        &self.basedir
291    }
292
293    fn has_package(&self, package_name : &str) -> bool
294    {
295        let mut path = PathBuf::from(&self.basedir);
296        path.push(package_name);
297
298        path.exists()
299    }
300
301    fn list_packages(&mut self) -> Vec<String>
302    {
303        if let Some(package_list) = &self.package_list
304        {
305            return package_list.clone();
306        }
307
308        let mut package_list : Vec<String> = Vec::new();
309        let path = PathBuf::from(&self.basedir);
310
311        let walkdir = WalkDir::new(path).min_depth(1).max_depth(1).into_iter();
312
313        for dir_entry in walkdir
314        {
315            if let Ok(entry) = dir_entry
316            {
317                if !entry.file_type().is_dir()
318                {
319                    continue;
320                }
321
322                package_list.push(entry.file_name().to_string_lossy().to_string());
323            }
324        }
325
326        package_list.sort();
327
328        self.package_list = Some(package_list.clone());
329
330        package_list
331    }
332
333    fn read_file<'a>(&'a mut self, package_name: &str, pathname: &str) -> Result<Box<dyn ReadSeek + 'a>, PackageError>
334    {
335        let mut path = PathBuf::from(&self.basedir);
336        path.push(format!("{}/{}", package_name, pathname));
337
338        let file = BufReader::new(File::open(path).map_err(
339            |error| PackageError::IoError(error))?);
340
341        Ok(Box::new(file))
342    }
343
344    fn write_file<'a>(&'a mut self, package_name : &str, pathname : &str) -> Result<Box<dyn Write + 'a>, PackageError>
345    {
346        let mut path = PathBuf::from(&self.basedir);
347        path.push(format!("{}/{}", package_name, pathname));
348        let mut folder_path = path.clone();
349
350        if !pathname.ends_with("/")
351        {
352            folder_path.pop();
353        }
354
355        std::fs::create_dir_all(folder_path).map_err(
356            |error| PackageError::IoError(error))?;
357
358        let file = BufWriter::new(File::create(path).map_err(
359            |error| PackageError::IoError(error))?);
360
361        Ok(Box::new(file))
362    }
363
364    fn iter_entries<'a>(&'a mut self, package_name : &str, type_folder : &str) -> Box<dyn Iterator<Item = Result<String, PackageError>> + 'a>
365    {
366        let mut path = PathBuf::from(&self.basedir);
367        path.push(package_name);
368        path.push(type_folder);
369
370        Box::new(FilesystemIter::new(path))
371    }
372
373    fn trust_level(&self, _package_name: &str) -> TrustLevel
374    {
375        self.trust
376    }
377}
378
379fn package_error_for_zip_error(error : ZipError) -> PackageError
380{
381    return match error
382    {
383        ZipError::Io(io_error) => { PackageError::IoError(io_error) }
384        ZipError::InvalidArchive(error_msg) => { PackageError::BadData(error_msg.to_string()) }
385        ZipError::UnsupportedArchive(error_msg) => { PackageError::BadData(error_msg.to_string()) }
386        ZipError::FileNotFound => { PackageError::DataNotFound }
387    }
388}
389
390struct FakeSeekReader<R : Read>
391{
392    reader : R
393}
394
395impl<R : Read> FakeSeekReader<R>
396{
397    fn new(reader : R) -> FakeSeekReader<R>
398    {
399        FakeSeekReader
400        {
401            reader
402        }
403    }
404}
405
406impl<R : Read> Read for FakeSeekReader<R>
407{
408    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize>
409    {
410        self.reader.read(buf)
411    }
412}
413
414impl<R : Read> Seek for FakeSeekReader<R>
415{
416    fn seek(&mut self, _pos: SeekFrom) -> std::io::Result<u64>
417    {
418        Err(std::io::Error::new(std::io::ErrorKind::Other, "Seek not supported on ZIP archives"))
419    }
420}
421
422struct ZipFolderIter<'a, T : Iterator<Item = &'a str>>
423{
424    iter : T,
425    type_folder : String
426}
427
428impl<'a, T : Iterator<Item = &'a str>> ZipFolderIter<'a, T>
429{
430    fn new(iter : T, type_folder : &str) -> ZipFolderIter<'a, T>
431    {
432        ZipFolderIter
433        {
434            iter,
435            type_folder : type_folder.to_string()
436        }
437    }
438}
439
440impl<'a, T : Iterator<Item = &'a str>> Iterator for ZipFolderIter<'a, T>
441{
442    type Item = Result<String, PackageError>;
443
444    fn next(&mut self) -> Option<Result<String, PackageError>>
445    {
446        while let Some(item) = self.iter.next()
447        {
448            if let Some(filename) = item.strip_prefix(&format!("{}/", self.type_folder))
449            {
450                return Some(Ok(filename.to_string()));
451            }
452        }
453
454        None
455    }
456}
457
458/// A [Source] that loads packages from zip/pk3 files in the filesystem. This is a good source to
459///  use for release builds. Each zip/pk3 file is a different package.
460pub struct ZipFolderSource
461{
462    basedir : String,
463    extension : String,
464    package_list : Option<Vec<String>>,
465    loaded_zips : HashMap<String, ZipArchive<BufReader<File>>, BuildHasherDefault<FxHasher>>,
466    trust : TrustLevel
467}
468
469impl ZipFolderSource
470{
471    /// Creates a new [ZipFolderSource] using the given directory to look for packages in.
472    pub fn new(directory : &str, trust : TrustLevel) -> ZipFolderSource
473    {
474        ZipFolderSource
475        {
476            basedir : directory.to_string(),
477            extension : String::from(".zip"),
478            package_list : None,
479            loaded_zips : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default()),
480            trust
481        }
482    }
483
484    /// Creates a new [ZipFolderSource] using the given directory to look for packages in, and
485    ///  the given file extension to look for zip files by package name. By default, new() will
486    ///  use "zip" as the extension.
487    pub fn with_extension(directory : &str, extension : &str, trust : TrustLevel) -> ZipFolderSource
488    {
489        let mut dot_extension = String::from(".");
490        dot_extension.push_str(extension);
491
492        ZipFolderSource
493        {
494            basedir : directory.to_string(),
495            extension : dot_extension,
496            package_list : None,
497            loaded_zips : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default()),
498            trust
499        }
500    }
501
502    /// Returns true if there is an open file handle to the zip file for the given package name.
503    pub fn zip_loaded(&self, package_name : &str) -> bool
504    {
505        self.loaded_zips.contains_key(package_name)
506    }
507
508    /// Opens an OS file handle to the zip file for the given package name.
509    pub fn load_zip(&mut self, package_name : &str) -> Result<(), PackageError>
510    {
511        if self.zip_loaded(package_name)
512        {
513            return Ok(());
514        }
515
516        let mut path = PathBuf::from(&self.basedir);
517        let mut zip_name = String::from(package_name);
518        zip_name.push_str(&self.extension);
519        path.push(zip_name);
520
521        let file = BufReader::new(File::open(path).map_err(
522            |error| PackageError::IoError(error))?);
523
524        let archive = ZipArchive::new(file).map_err(
525            |error| package_error_for_zip_error(error))?;
526
527        self.loaded_zips.insert(package_name.to_string(), archive);
528
529        Ok(())
530    }
531
532    /// Closes the OS file handle to the zip file for the given package name.
533    pub fn unload_zip(&mut self, package_name : &str)
534    {
535        self.loaded_zips.remove(package_name);
536    }
537}
538
539impl Source for ZipFolderSource
540{
541    fn get_uri(&self) -> &str
542    {
543        &self.basedir
544    }
545
546    fn has_package(&self, package_name: &str) -> bool
547    {
548        let mut path = PathBuf::from(&self.basedir);
549        let mut zip_name = String::from(package_name);
550        zip_name.push_str(&self.extension);
551        path.push(zip_name);
552
553        path.exists()
554    }
555
556    fn list_packages(&mut self) -> Vec<String>
557    {
558        if let Some(package_list) = &self.package_list
559        {
560            return package_list.clone();
561        }
562
563        let mut package_list: Vec<String> = Vec::new();
564        let path = PathBuf::from(&self.basedir);
565
566        let walkdir = WalkDir::new(path).min_depth(1).max_depth(1).into_iter();
567
568        for dir_entry in walkdir
569        {
570            if let Ok(entry) = dir_entry
571            {
572                if !entry.file_type().is_file()
573                {
574                    continue;
575                }
576
577                if let Some(stripped) = entry.file_name().to_string_lossy()
578                    .strip_suffix(&self.extension)
579                {
580                    package_list.push(stripped.to_string());
581                }
582            }
583        }
584
585        package_list.sort();
586
587        self.package_list = Some(package_list.clone());
588
589        package_list
590    }
591
592    fn read_file<'a>(&'a mut self, package_name: &str, pathname: &str) -> Result<Box<dyn ReadSeek + 'a>, PackageError>
593    {
594        self.load_zip(package_name)?;
595
596        if let Some(archive) = self.loaded_zips.get_mut(package_name)
597        {
598            match archive.by_name(pathname)
599            {
600                Ok(reader) => { return Ok(Box::new(FakeSeekReader::new(reader))); }
601                Err(error) => { return Err(package_error_for_zip_error(error)); }
602            }
603        }
604
605        Err(PackageError::IoError(std::io::Error::new(std::io::ErrorKind::NotFound, "Archive not found")))
606    }
607
608    fn iter_entries<'a>(&'a mut self, package_name: &str, type_folder: &str) -> Box<dyn Iterator<Item=Result<String, PackageError>> + 'a>
609    {
610        if self.load_zip(package_name).is_err()
611        {
612            return Box::new(EmptyEntryIter::new());
613        }
614
615        let mut iter : Box<dyn Iterator<Item=Result<String, PackageError>>> = Box::new(EmptyEntryIter::new());
616
617        if let Some(archive) = self.loaded_zips.get(package_name)
618        {
619            let archive_iter = archive.file_names();
620
621            iter = Box::new(ZipFolderIter::new(archive_iter, type_folder));
622        }
623
624        iter
625    }
626
627    fn trust_level(&self, _package_name: &str) -> TrustLevel
628    {
629        self.trust
630    }
631}
keeshond_datapack/source.rs

keeshond_datapack/
source.rs
