rapid_fs/
vfs.rs

1use std::collections::{HashMap, VecDeque};
2use std::fmt::{Debug, Formatter};
3use std::fs;
4use std::fs::{File, OpenOptions};
5use std::io::{Read, Seek, SeekFrom, Write};
6use std::path::{Path, PathBuf};
7use std::sync::Arc;
8
9use bytes::BufMut;
10use log::warn;
11use serde::Deserialize;
12use thiserror::Error;
13
14pub const DOMAINS_SUBDIR: &str = "domains";
15pub const RESOURCES_SUBDIR: &str = "files";
16pub const TMP_SUBDIR: &str = ".tmp";
17pub const VERSIONS_SUBDIR: &str = "versions";
18pub const DRAFTS_SUBDIR: &str = "drafts";
19pub const ECMA_SUBDIR: &str = "ecma";
20pub const PLUGINS_SUBDIR: &str = "plugins";
21
22pub type Result<T> = std::result::Result<T, VfsErr>;
23
24#[derive(Debug, Error)]
25pub enum VfsErr {
26    #[error("Domain not found - {0}")]
27    Domain(String),
28    #[error("File not found - {0}")]
29    FileNotFound(String),
30    #[error("Schema file not found - {0}")]
31    SchemaFileNotFound(String),
32    #[error("Absolute file paths not supported - {0}")]
33    AbsolutePathNotSupported(String),
34    #[error("Dot paths not supported - {0}")]
35    DotPathsNotSupported(String),
36    #[error("Error parsing JSON - {0}")]
37    JsonErr(serde_json::Error),
38    #[error("IO error - {0}")]
39    Io(std::io::Error),
40    #[error("IO error - {0}")]
41    StripPrefixErr(std::path::StripPrefixError),
42    #[error("IO error - {0}")]
43    Utf8(std::string::FromUtf8Error),
44}
45
46#[derive(Debug, Deserialize)]
47pub struct DomainOptions {
48    pub service_id: i64,
49    pub version: String,
50    pub is_draft: bool,
51}
52
53///[Vfs] i.e. virtual file system is specifically designed to constrain access to the file system via API requests
54/// whilst also making the access mechanism abstract away from the low level OS FS APIs.
55/// Specifically [Vfs] is written to provide access to a structure which assumes multiple APIs are served from a single root directory.
56///
57/// [BoundVfs] is then used to ensure that when one of these are accessed, all API calls are bound to the specific service ID bounded by it.
58/// The underlying filesystem follows this basic format:
59/// ```yaml
60/// services:
61///   domains:
62///     my-api.apps.hypi.app - file name is the domain, line 1 is the service ID, line 2 is the version
63///   service1:
64///     files:
65///       file1.png - the static files are served from here but permission checks are done before serving
66///       file2.txt
67///     versions:
68///       v1
69///       v2
70///       v3
71///         schema.xml - required
72///         endpoint1.xml
73///         endpoint2.xml
74///         table1.xml
75///         table2.xml
76/// ```
77pub trait Vfs: Sync + Send {
78    ///A base directory against which all paths are [resolve]d.
79    fn root(&self) -> &PathBuf;
80    fn resolve(&self, child: &str) -> Result<PathBuf> {
81        let root = self.root();
82        let child_path = Path::new(child);
83        //VERY important - root.join below is not safe if child is absolute
84        //because join replaces root with child if child is absolute
85        if child_path.is_absolute() {
86            Err(VfsErr::AbsolutePathNotSupported(child.to_owned()))
87        } else if child.contains("./") || child.contains("..") {
88            Err(VfsErr::DotPathsNotSupported(child.to_owned()))
89        } else {
90            let resolved = root.join(child_path);
91            let res_str = resolved.to_string_lossy().to_string();
92            //note we don't call resolved.canonicalize() because we don't want to hit the file system
93            //this resolve is used in all implementations of the Vfs which is not necessarily resolved from disk
94            if res_str.starts_with(&root.to_string_lossy().to_string())
95            /* root.to_string_lossy().to_string().contains(&res_str)*/
96            {
97                Ok(resolved)
98            } else {
99                //somehow the resolved path broke out from under root, don't allow it to continue
100                Err(VfsErr::DotPathsNotSupported(child.to_owned()))
101            }
102        }
103    }
104    fn domain_file(&self, domain: &str) -> Result<PathBuf> {
105        self.resolve(format!("{}/{}", DOMAINS_SUBDIR, domain).as_str())
106    }
107    fn resource_dir(&self, service_id: i64) -> Result<PathBuf> {
108        let dir = self.resolve(format!("{}/{}", service_id, RESOURCES_SUBDIR).as_str())?;
109        fs::create_dir_all(dir.clone()).map_err(VfsErr::Io)?;
110        Ok(dir)
111    }
112    fn plugins_dir(&self, service_id: i64) -> Result<PathBuf> {
113        let dir = self.resolve(format!("{}/{}", service_id, PLUGINS_SUBDIR).as_str())?;
114        fs::create_dir_all(dir.clone()).map_err(VfsErr::Io)?;
115        Ok(dir)
116    }
117    fn tmp_dir(&self, service_id: i64) -> Result<PathBuf> {
118        let dir = self.resolve(format!("{}/{}", service_id, TMP_SUBDIR).as_str())?;
119        fs::create_dir_all(dir.clone()).map_err(VfsErr::Io)?;
120        Ok(dir)
121    }
122    fn resource_file(&self, service_id: i64, name: &str) -> Result<PathBuf> {
123        let mut path = self.resource_dir(service_id)?;
124        path.push(name);
125        Ok(path)
126    }
127    fn schema_file(&self, service_id: i64, is_draft: bool, version: &str, file: &str) -> Result<PathBuf> {
128        self.resolve(format!("{}/{}/{}/{}", service_id, if is_draft { DRAFTS_SUBDIR } else { VERSIONS_SUBDIR }, version, file).as_str())
129    }
130    fn ecma_dir(&self, service_id: i64, is_draft: bool, version: &str) -> Result<PathBuf> {
131        self.resolve(
132            format!(
133                "{}/{}/{}/{}",
134                service_id, if is_draft { DRAFTS_SUBDIR } else { VERSIONS_SUBDIR }, version, ECMA_SUBDIR
135            )
136                .as_str(),
137        )
138    }
139    fn read(&self, file: PathBuf) -> Result<Box<dyn Read + '_>>;
140    fn open_with(&self, file: PathBuf, opts: OpenOptions) -> Result<Box<dyn VfsFile>>;
141    fn read_domain_file(&self, domain: &str) -> Result<DomainOptions> {
142        match self.domain_file(domain) {
143            Ok(file) => {
144                let mut data = vec![];
145                let mut input = self.read(file)?;
146                let mut buffer = [0; 1024];
147                while let Ok(n) = input.read(&mut buffer).map_err(VfsErr::Io) {
148                    if n == 0 {
149                        break;
150                    }
151                    data.extend_from_slice(&buffer[0..n]);
152                }
153                Ok(serde_json::from_slice(&data).map_err(VfsErr::JsonErr)?)
154            }
155            Err(e) => Err(e),
156        }
157    }
158    fn read_resource_file(&self, service_id: i64, filename: &str) -> Result<Box<dyn Read + '_>> {
159        match self.resource_file(service_id, filename) {
160            Ok(file) => self.read(file),
161            Err(e) => Err(e),
162        }
163    }
164    fn read_schema_file(&self, service_id: i64, is_draft: bool, version: &str, filename: &str) -> Result<String> {
165        match self.schema_file(service_id, is_draft, version, filename) {
166            Ok(file) => {
167                let mut data = vec![];
168                let mut input = self.read(file)?;
169                let mut buffer = [0; 1024];
170                while let Ok(n) = input.read(&mut buffer).map_err(VfsErr::Io) {
171                    if n == 0 {
172                        break;
173                    }
174                    data.extend_from_slice(&buffer[0..n]);
175                }
176                Ok(String::from_utf8(data).map_err(VfsErr::Utf8)?)
177            }
178            Err(e) => Err(e),
179        }
180    }
181    fn read_ecma<'a>(&'a self, service_id: i64, is_draft: bool, version: &str) -> Result<DirStream<'a, Self>> {
182        let dir = self.ecma_dir(service_id, is_draft, version)?;
183        self.dir_stream(dir)
184    }
185    fn dir_stream<'a>(&'a self, dir: PathBuf) -> Result<DirStream<'a, Self>> {
186        if dir.to_string_lossy().contains("..") {
187            warn!("ECMA script path cannot contain '..' i.e. must be absolute, full path");
188            return Err(VfsErr::DotPathsNotSupported(format!(
189                "ECMA script path can't have .. in {}",
190                dir.to_string_lossy()
191            )));
192        }
193        match self.read_dir(&dir) {
194            Ok(read_dir) => {
195                let mut stream: DirStream<'a, Self> = DirStream {
196                    base: dir,
197                    buf: VecDeque::new(),
198                    vfs: self,
199                };
200                stream.buf.push_back(read_dir);
201                Ok(stream)
202            }
203            Err(e) => Err(e),
204        }
205    }
206    fn read_dir(&self, dir: &PathBuf) -> Result<VirtualReadDir>;
207}
208
209pub struct VirtualReadDir {
210    inner: Box<dyn Iterator<Item=PathBuf>>,
211}
212
213impl Iterator for VirtualReadDir {
214    type Item = PathBuf;
215
216    fn next(&mut self) -> Option<Self::Item> {
217        self.inner.next()
218    }
219}
220
221pub struct DirStream<'a, F>
222    where
223        F: Vfs + ?Sized,
224{
225    base: PathBuf,
226    buf: VecDeque<VirtualReadDir>,
227    vfs: &'a F,
228}
229
230impl<'a, F: Vfs> Iterator for DirStream<'a, F> {
231    type Item = Result<(PathBuf, PathBuf)>;
232
233    fn next(&mut self) -> Option<Self::Item> {
234        if let Some(dir) = self.buf.back_mut() {
235            if let Some(path) = dir.next() {
236                //can't use canonicalize because it goes to the filesystem
237                // let path = match path.canonicalize().map_err(VfsErr::Io) {
238                //     Ok(p) => p,
239                //     Err(e) => return Some(Err(e)),
240                // };
241                if path.to_string_lossy().contains("..") {
242                    warn!(
243                        "Skipping path {} because it contains '..'",
244                        path.to_string_lossy()
245                    );
246                    return self.next();
247                }
248                if path.is_dir() {
249                    match self.vfs.read_dir(&path) {
250                        Ok(child) => {
251                            self.buf.push_front(child);
252                            self.next()
253                        }
254                        Err(e) => Some(Err(e)),
255                    }
256                } else {
257                    if path.starts_with(&self.base) {
258                        let filename = match path
259                            .strip_prefix(&self.base)
260                            .map_err(VfsErr::StripPrefixErr)
261                        {
262                            Ok(p) => p,
263                            Err(e) => return Some(Err(e)),
264                        };
265                        Some(Ok((filename.to_owned(), path)))
266                    } else {
267                        //Some(Err(VfsErr::Generic(format!())))
268                        self.next() //silently skip files that are not in the service's base directory
269                    }
270                }
271            } else {
272                self.buf.pop_back();
273                self.next()
274            }
275        } else {
276            None
277        }
278    }
279}
280
281#[derive(Clone)]
282pub struct FilesystemVfs {
283    ///The absolute path to the directory where the services are kept
284    ///This is important because we ensure that all operations are a sub-directory of this
285    services_dir: PathBuf,
286}
287
288pub trait VfsFile: Read + Write + Seek {
289    fn path(&self) -> PathBuf;
290    fn clone(&self) -> Result<Box<dyn VfsFile>>;
291}
292
293impl dyn VfsFile {
294    pub fn save_to<F>(&self, fs: Arc<BoundVfs<F>>, new_name: Option<String>) -> Result<String>
295        where
296            F: Vfs,
297    {
298        fs.save_to(self, new_name)
299    }
300    pub fn discard<F>(&self, fs: Arc<BoundVfs<F>>) -> Result<()>
301        where
302            F: Vfs,
303    {
304        fs.discard(self)
305    }
306}
307
308impl Debug for dyn VfsFile {
309    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
310        f.write_str("VfsFile")
311    }
312}
313
314pub struct VfsFileSystemFile(File, PathBuf);
315
316impl VfsFile for VfsFileSystemFile {
317    fn path(&self) -> PathBuf {
318        self.1.clone()
319    }
320    fn clone(&self) -> Result<Box<dyn VfsFile>> {
321        let mut opts = OpenOptions::new();
322        opts.read(true);
323        Ok(Box::new(VfsFileSystemFile(
324            opts.open(self.1.clone()).map_err(VfsErr::Io)?,
325            self.1.clone(),
326        )))
327    }
328}
329
330impl Read for VfsFileSystemFile {
331    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
332        self.0.read(buf)
333    }
334}
335
336impl Write for VfsFileSystemFile {
337    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
338        self.0.write(buf)
339    }
340
341    fn flush(&mut self) -> std::io::Result<()> {
342        self.0.flush()
343    }
344}
345
346impl Seek for VfsFileSystemFile {
347    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
348        self.0.seek(pos)
349    }
350}
351
352impl Vfs for FilesystemVfs {
353    fn root(&self) -> &PathBuf {
354        &self.services_dir
355    }
356
357    fn read(&self, file: PathBuf) -> Result<Box<dyn Read + '_>> {
358        if file.to_string_lossy().contains("..") {
359            return Err(VfsErr::DotPathsNotSupported(format!(
360                "Cannot read file with .. in path {}",
361                file.to_string_lossy()
362            )));
363        }
364        Ok(Box::new(File::open(file).map_err(VfsErr::Io)?))
365    }
366    fn open_with(&self, path: PathBuf, opts: OpenOptions) -> Result<Box<dyn VfsFile>> {
367        if path.to_string_lossy().contains("..") {
368            return Err(VfsErr::DotPathsNotSupported(format!(
369                "Cannot open file with .. in path {}",
370                path.to_string_lossy()
371            )));
372        }
373        let file = opts.open(path.clone()).map_err(VfsErr::Io)?;
374        Ok(Box::new(VfsFileSystemFile(file, path)))
375    }
376
377    fn read_dir(&self, dir: &PathBuf) -> Result<VirtualReadDir> {
378        if dir.to_string_lossy().contains("..") {
379            return Err(VfsErr::DotPathsNotSupported(format!(
380                "Cannot read dir with .. in path {}",
381                dir.to_string_lossy()
382            )));
383        }
384        let it = fs::read_dir(dir).map_err(VfsErr::Io)?;
385        let it = it.map(|v| v.map(|e| e.path())).flatten();
386        let it: Box<dyn Iterator<Item=PathBuf>> = Box::new(it);
387        Ok(VirtualReadDir { inner: it })
388    }
389}
390
391impl FilesystemVfs {
392    pub fn new(services_dir: String) -> Self {
393        FilesystemVfs {
394            services_dir: PathBuf::from(services_dir),
395        }
396    }
397}
398
399#[allow(unused)]
400pub struct MemVfsFile {
401    path: PathBuf,
402    data: Vec<u8>,
403    offset: usize,
404}
405
406impl Seek for MemVfsFile {
407    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
408        match pos {
409            SeekFrom::Start(_start) => {}
410            SeekFrom::End(_end) => {}
411            SeekFrom::Current(_current) => {}
412        }
413        todo!();
414        // Ok(0)
415    }
416}
417
418impl Read for MemVfsFile {
419    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
420        let start = self.offset;
421        let mut end = start + buf.len();
422        let buf_len = self.data.len();
423        if end >= buf_len {
424            end = buf_len;
425        }
426        if start >= end {
427            return Ok(0);
428        }
429        let slice = &self.data[start..end];
430        let read = end - start;
431        buf[0..read].clone_from_slice(slice);
432        self.offset = end;
433        Ok(read)
434    }
435}
436
437impl Write for MemVfsFile {
438    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
439        self.data.put_slice(buf);
440        Ok(buf.len())
441    }
442
443    fn flush(&mut self) -> std::io::Result<()> {
444        println!(
445            "MemVfsFile::flush:{}",
446            String::from_utf8(self.data.clone()).unwrap()
447        );
448        Ok(())
449    }
450}
451
452impl VfsFile for MemVfsFile {
453    fn path(&self) -> PathBuf {
454        self.path.clone()
455    }
456    fn clone(&self) -> Result<Box<dyn VfsFile>> {
457        Ok(Box::new(MemVfsFile {
458            path: self.path.clone(),
459            data: self.data.clone(),
460            offset: 0,
461        }))
462    }
463}
464
465#[derive(Clone)]
466pub struct MemoryVfs {
467    pub root: PathBuf,
468    pub data: HashMap<String, String>,
469}
470
471impl Vfs for MemoryVfs {
472    fn root(&self) -> &PathBuf {
473        &self.root
474    }
475
476    fn read(&self, file: PathBuf) -> Result<Box<dyn Read + '_>> {
477        if file.to_string_lossy().contains("..") {
478            return Err(VfsErr::DotPathsNotSupported(format!(
479                "Cannot read file with .. in path {}",
480                file.to_string_lossy()
481            )));
482        }
483        match self.data.get(file.to_string_lossy().as_ref()) {
484            Some(data) => {
485                let data: &[u8] = data.as_bytes();
486                Ok(Box::new(data))
487            }
488            None => Err(VfsErr::FileNotFound(format!(
489                "File not found - {}",
490                file.to_string_lossy()
491            ))),
492        }
493    }
494
495    fn open_with(&self, file: PathBuf, _opts: OpenOptions) -> Result<Box<dyn VfsFile>> {
496        if file.to_string_lossy().contains("..") {
497            return Err(VfsErr::DotPathsNotSupported(format!(
498                "Cannot read file with .. in path {}",
499                file.to_string_lossy()
500            )));
501        }
502        match self.data.get(file.to_string_lossy().as_ref()) {
503            Some(data) => {
504                let data: &[u8] = data.as_bytes();
505                Ok(Box::new(MemVfsFile {
506                    path: file,
507                    data: Vec::from(data),
508                    offset: 0,
509                }))
510            }
511            None => {
512                //we assume write/append and create it - means there's a different behaviour with in-memory vs disk
513                Ok(Box::new(MemVfsFile {
514                    path: file,
515                    data: vec![],
516                    offset: 0,
517                }))
518                // Err(VfsErr::FileNotFound(format!(
519                //     "File not found - {}",
520                //     file.to_string_lossy()
521                // )))
522            }
523        }
524    }
525
526    fn read_dir(&self, dir: &PathBuf) -> Result<VirtualReadDir> {
527        if dir.to_string_lossy().contains("..") {
528            return Err(VfsErr::DotPathsNotSupported(format!(
529                "Cannot read dir with .. in path {}",
530                dir.to_string_lossy()
531            )));
532        }
533        let it: Vec<_> = self
534            .data
535            .keys()
536            .map(PathBuf::from)
537            .skip_while(|path| !path.starts_with(dir))
538            .collect();
539        Ok(VirtualReadDir {
540            inner: Box::new(it.into_iter()),
541        })
542    }
543}
544
545pub struct BoundVfs<F>
546    where
547        F: Vfs,
548{
549    pub options: DomainOptions,
550    pub vfs: Arc<F>,
551}
552
553impl<F> BoundVfs<F>
554    where
555        F: Vfs,
556{
557    pub fn new(options: DomainOptions, vfs: Arc<F>) -> BoundVfs<F> {
558        Self { options, vfs }
559    }
560    pub fn read_schema_file(&self, name: &str) -> Result<String> {
561        self.vfs
562            .read_schema_file(self.options.service_id, self.options.is_draft, self.options.version.as_str(), name)
563    }
564
565    pub fn ecma_files(&self) -> Result<DirStream<F>> {
566        self.vfs
567            .read_ecma(self.options.service_id, self.options.is_draft, self.options.version.as_str())
568    }
569
570    pub fn read_ecma_file(&self, mut file: PathBuf) -> Result<String> {
571        if file.starts_with("./") {
572            file = file
573                .strip_prefix("./")
574                .map_err(VfsErr::StripPrefixErr)?
575                .to_owned();
576        }
577        let mut path = self
578            .vfs
579            .ecma_dir(self.options.service_id, self.options.is_draft, self.options.version.as_str())?;
580        path.push(file);
581        let mut read = self.vfs.read(path)?;
582        let mut str = String::new();
583        read.read_to_string(&mut str).map_err(VfsErr::Io)?;
584        Ok(str)
585    }
586
587    pub fn resource_dir(&self) -> Result<PathBuf> {
588        self.vfs.resource_dir(self.options.service_id)
589    }
590
591    pub fn resolve_resource(&self, mut file: PathBuf) -> Result<PathBuf> {
592        if file.starts_with("./") {
593            file = file
594                .strip_prefix("./")
595                .map_err(VfsErr::StripPrefixErr)?
596                .to_owned();
597        } else if file.to_string_lossy().contains("..") {
598            return Err(VfsErr::DotPathsNotSupported(format!(
599                "Cannot open file with .. in path {}",
600                file.to_string_lossy()
601            )));
602        }
603        let mut path = self.vfs.resource_dir(self.options.service_id)?;
604        path.push(file);
605        Ok(path)
606    }
607    pub fn resolve_plugin(&self, mut file: PathBuf) -> Result<PathBuf> {
608        if file.starts_with("./") {
609            file = file
610                .strip_prefix("./")
611                .map_err(VfsErr::StripPrefixErr)?
612                .to_owned();
613        } else if file.to_string_lossy().contains("..") {
614            return Err(VfsErr::DotPathsNotSupported(format!(
615                "Cannot open file with .. in path {}",
616                file.to_string_lossy()
617            )));
618        }
619        let mut path = self.vfs.plugins_dir(self.options.service_id)?;
620        path.push(file);
621        Ok(path)
622    }
623    pub fn open(&self, mut file: PathBuf, opts: OpenOptions) -> Result<Box<dyn VfsFile>> {
624        if file.starts_with("./") {
625            file = file
626                .strip_prefix("./")
627                .map_err(VfsErr::StripPrefixErr)?
628                .to_owned();
629        } else if file.to_string_lossy().contains("..") {
630            return Err(VfsErr::DotPathsNotSupported(format!(
631                "Cannot open file with .. in path {}",
632                file.to_string_lossy()
633            )));
634        }
635        self.vfs.open_with(self.resolve_resource(file)?, opts)
636    }
637
638    pub fn discard<I>(&self, _file: &I) -> Result<()>
639        where
640            I: VfsFile + ?Sized,
641    {
642        todo!();
643        // Ok(())
644    }
645    pub fn save_to<I>(&self, file: &I, new_name: Option<String>) -> Result<String>
646        where
647            I: VfsFile + ?Sized,
648    {
649        let mut other_path = file.path();
650        let mut path = self.vfs.resource_dir(self.options.service_id)?;
651        if other_path.starts_with(&path) {
652            other_path = PathBuf::from(
653                other_path
654                    .strip_prefix(&path)
655                    .map_err(VfsErr::StripPrefixErr)?,
656            );
657        }
658        if other_path.starts_with(TMP_SUBDIR) {
659            other_path = PathBuf::from(
660                other_path
661                    .strip_prefix(TMP_SUBDIR)
662                    .map_err(VfsErr::StripPrefixErr)?,
663            )
664        }
665        path.push(other_path);
666        if let Some(file_name) = new_name {
667            path.set_file_name(file_name);
668        }
669        let name = if let Some(name) = path.file_name().map(|v| v.to_str()).flatten() {
670            name.to_string()
671        } else {
672            file.path()
673                .to_string_lossy()
674                .split("/")
675                .last()
676                .unwrap()
677                .to_string()
678        };
679        fs::rename(file.path(), path).map_err(VfsErr::Io)?;
680        Ok(name)
681    }
682}
rapid_fs/vfs.rs

rapid_fs/
vfs.rs
