web_scrape/cache/
web_cache.rs

1use crate::Error;
2use clerr::{Code, Report};
3use enc::base_64::Base64Encoder;
4use enc::hex::HexEncoder;
5use enc::StringEncoder;
6use file_storage::{FilePath, FolderPath};
7use web_url::WebUrl;
8
9/// Responsible for caching web data.
10#[derive(Clone, Debug)]
11pub struct WebCache {
12    local: Option<FolderPath>,
13    remote: Option<FolderPath>,
14    encoder: Base64Encoder,
15    extension: String,
16}
17
18impl WebCache {
19    //! Construction
20
21    /// Creates a new web cache.
22    pub fn new(local: Option<FolderPath>, remote: Option<FolderPath>) -> Self {
23        Self {
24            local,
25            remote,
26            encoder: Base64Encoder::url_safe_encoder(),
27            extension: ".web-cache".to_string(),
28        }
29    }
30}
31
32impl WebCache {
33    //! Read
34
35    /// Reads the optional cached data for the `url`.
36    pub fn read(&self, url: &WebUrl) -> Result<Option<Vec<u8>>, Error> {
37        if let Some(local) = &self.local {
38            let file: FilePath = self.file_for_root(url, local)?;
39            if let Some(data) = file.read_as_vec_if_exists()? {
40                return Ok(Some(data));
41            }
42        }
43
44        if let Some(remote) = &self.remote {
45            let file: FilePath = self.file_for_root(url, remote)?;
46            if let Some(data) = file.read_as_vec_if_exists()? {
47                return Ok(Some(data));
48            }
49        }
50
51        Ok(None)
52    }
53}
54
55impl WebCache {
56    //! Write
57
58    /// Overwrites the cached `data` for the `url`.
59    pub fn write(&self, url: &WebUrl, data: &[u8]) -> Result<(), Error> {
60        if let Some(local) = &self.local {
61            self.write_to_root(url, data, local)?;
62        }
63        if let Some(remote) = &self.remote {
64            self.write_to_root(url, data, remote)?;
65        }
66        Ok(())
67    }
68
69    /// Writes the `data` from the `url` to the `root` folder.
70    fn write_to_root(&self, url: &WebUrl, data: &[u8], root: &FolderPath) -> Result<(), Error> {
71        let file: FilePath = self.file_for_root(url, root)?;
72        file.delete()?;
73        Ok(file.write_data_if_not_exists(data).map(|_| ())?)
74    }
75}
76
77impl WebCache {
78    //! Files
79
80    /// Gets the file for the `url` given the `root` folder.
81    fn file_for_root(&self, url: &WebUrl, root: &FolderPath) -> Result<FilePath, Error> {
82        let folder_char: char = self.folder_char(url.as_str());
83        let base_64: String = self
84            .encoder
85            .encode_as_string(url.as_str().as_bytes())
86            .map_err(|e| {
87                Error::Other(Report::new(Code::error(
88                    "WEB_CACHE_BASE_64",
89                    format!("error converting the URL to base-64: {} -- {}", url, e),
90                )))
91            })?;
92        let extra: usize = folder_char.len_utf8()
93            + root.path().file_separator().len_utf8()
94            + base_64.len()
95            + self.extension.len();
96        root.clone_with_extra_capacity(extra)
97            .with_appended_char(folder_char)
98            .make_folder()
99            .with_appended(base_64.as_str())
100            .make_file(self.extension.as_str())
101            .map_err(|path| {
102                Error::Other(Report::new(Code::error(
103                    "WEB_CACHE_INVALID_EXTENSION",
104                    format!("the file extension makes the path a folder: {}", path),
105                )))
106            })
107    }
108
109    /// Gets the folder char for the cache `key`. (a single lowercase hex char)
110    fn folder_char(&self, key: &str) -> char {
111        let mut fold: u8 = 0;
112        for byte in key.as_bytes() {
113            fold ^= byte;
114        }
115        let (_, hex) = HexEncoder::LOWER.encode_chars(fold);
116        hex
117    }
118}
web_scrape/cache/web_cache.rs

web_scrape/cache/
web_cache.rs
