glowpub/
cached.rs

1use std::{
2    collections::BTreeSet,
3    error::Error,
4    path::{Path, PathBuf},
5    str::FromStr,
6};
7
8use mime::Mime;
9use reqwest::header::CONTENT_TYPE;
10use serde::{de::DeserializeOwned, Serialize};
11
12use crate::{
13    api::{BoardPosts, GlowficError, PostInBoard, Replies},
14    types::{Continuity, Icon, Thread},
15    utils::{
16        extension_to_image_mime, guess_image_mime, http_client, mime_to_image_extension, url_hash,
17    },
18    Board, Post, Reply,
19};
20
21const CACHE_ROOT: &str = "./cache";
22
23impl Board {
24    fn cache_key(id: u64) -> PathBuf {
25        format!("{CACHE_ROOT}/boards/{id}.json").into()
26    }
27
28    pub async fn get_cached(
29        id: u64,
30        invalidate_cache: bool,
31    ) -> Result<Result<Self, Vec<GlowficError>>, Box<dyn Error>> {
32        get_cached_glowfic(&Self::url(id), &Self::cache_key(id), invalidate_cache).await
33    }
34}
35
36impl Post {
37    fn cache_key(id: u64) -> PathBuf {
38        format!("{CACHE_ROOT}/posts/{id}/post.json").into()
39    }
40
41    pub async fn get_cached(
42        id: u64,
43        invalidate_cache: bool,
44    ) -> Result<Result<Self, Vec<GlowficError>>, Box<dyn Error>> {
45        get_cached_glowfic(&Self::url(id), &Self::cache_key(id), invalidate_cache).await
46    }
47}
48
49impl Replies {
50    fn cache_key(id: u64) -> PathBuf {
51        format!("{CACHE_ROOT}/posts/{id}/replies.json").into()
52    }
53
54    pub async fn get_all_cached(
55        id: u64,
56        invalidate_cache: bool,
57    ) -> Result<Result<Vec<Reply>, Vec<GlowficError>>, Box<dyn Error>> {
58        let cache_path = Self::cache_key(id);
59
60        if !invalidate_cache {
61            if let Ok(data) = std::fs::read(&cache_path) {
62                let parsed: Result<Self, Vec<GlowficError>> =
63                    serde_json::from_slice(&data).unwrap();
64
65                if let Ok(replies) = parsed {
66                    return Ok(Ok(replies.0));
67                }
68            }
69        }
70
71        let response = Self::get_all(id).await?;
72
73        std::fs::create_dir_all(cache_path.parent().unwrap()).unwrap();
74        write_if_changed(&cache_path, serde_json::to_vec_pretty(&response).unwrap()).unwrap();
75
76        Ok(response)
77    }
78}
79
80impl BoardPosts {
81    fn cache_key(id: u64) -> PathBuf {
82        format!("{CACHE_ROOT}/boards/{id}/posts.json").into()
83    }
84
85    pub async fn get_all_cached(
86        id: u64,
87        invalidate_cache: bool,
88    ) -> Result<Result<Vec<PostInBoard>, Vec<GlowficError>>, Box<dyn Error>> {
89        let cache_path = Self::cache_key(id);
90
91        if !invalidate_cache {
92            if let Ok(data) = std::fs::read(&cache_path) {
93                let parsed: Result<Vec<PostInBoard>, Vec<GlowficError>> =
94                    serde_json::from_slice(&data).unwrap();
95
96                if let Ok(posts) = parsed {
97                    return Ok(Ok(posts));
98                }
99            }
100        }
101
102        let response = Self::get_all(id).await?;
103
104        std::fs::create_dir_all(cache_path.parent().unwrap()).unwrap();
105        write_if_changed(&cache_path, serde_json::to_vec_pretty(&response).unwrap()).unwrap();
106
107        Ok(response)
108    }
109}
110
111impl Icon {
112    fn cache_key(id: u64, extension: &str) -> PathBuf {
113        // Note: names starting with a number can be problematic in epubs.
114        format!("{CACHE_ROOT}/images/glowfic_{id}.{extension}").into()
115    }
116
117    pub async fn download_cached(
118        &self,
119        invalidate_cache: bool,
120    ) -> Result<(Mime, Vec<u8>), Box<dyn Error>> {
121        let Self { id, url, .. } = self;
122
123        let Some(url) = url else {
124            return Err("No url provided for this icon".into());
125        };
126
127        if !invalidate_cache {
128            if let Ok((mime, data)) = read_image_file(Self::cache_key(*id, "*")) {
129                return Ok((mime, data));
130            }
131        }
132
133        log::info!("Downloading icon {id} from {url}");
134
135        let (mime, data) = download_image(url).await?;
136
137        let mime = guess_image_mime(&data).unwrap_or(mime);
138
139        let extension = mime_to_image_extension(&mime).ok_or(format!("Invalid mime: {mime}"))?;
140
141        let cache_path = Self::cache_key(*id, &extension);
142        std::fs::create_dir_all(cache_path.parent().unwrap()).unwrap();
143        write_if_changed(cache_path, &data).unwrap();
144
145        Ok((mime, data))
146    }
147}
148
149impl Thread {
150    pub async fn get_cached(
151        id: u64,
152        invalidate_cache: bool,
153    ) -> Result<Result<Self, Vec<GlowficError>>, Box<dyn Error>> {
154        let post = match Post::get_cached(id, invalidate_cache).await? {
155            Ok(post) => post,
156            Err(errors) => return Ok(Err(errors)),
157        };
158        let replies = match Replies::get_all_cached(id, invalidate_cache).await? {
159            Ok(replies) => replies,
160            Err(errors) => return Ok(Err(errors)),
161        };
162
163        Ok(Ok(Self { post, replies }))
164    }
165    pub async fn cache_all_icons(&self, invalidate_cache: bool) {
166        let icons: BTreeSet<_> = self.icons().collect();
167
168        for icon in icons {
169            if let Err(e) = icon.download_cached(invalidate_cache).await {
170                log::info!("{e:?}");
171            }
172        }
173        for url in self.image_urls() {
174            if let Err(e) = download_cached_image(&url, invalidate_cache).await {
175                log::info!("{e:?}");
176            }
177        }
178    }
179}
180
181impl Continuity {
182    pub async fn get_cached(
183        id: u64,
184        invalidate_cache: bool,
185    ) -> Result<Result<Self, Vec<GlowficError>>, Box<dyn Error>> {
186        let board = match Board::get_cached(id, invalidate_cache).await? {
187            Ok(board) => board,
188            Err(errors) => return Ok(Err(errors)),
189        };
190        let threads = match BoardPosts::get_all_cached(id, invalidate_cache).await? {
191            Ok(board_posts) => {
192                let mut threads = vec![];
193                for p in board_posts {
194                    log::info!("Downloading post {} - {}", p.id, &p.subject);
195                    let thread = match Thread::get_cached(p.id, invalidate_cache).await? {
196                        Ok(thread) => thread,
197                        Err(e) => return Ok(Err(e)),
198                    };
199                    threads.push(thread);
200                }
201                threads
202            }
203            Err(errors) => return Ok(Err(errors)),
204        };
205
206        Ok(Ok(Self { board, threads }))
207    }
208    pub async fn cache_all_icons(&self, invalidate_cache: bool) {
209        let icons: BTreeSet<_> = self.threads.iter().flat_map(|t| t.icons()).collect();
210        for icon in icons {
211            if let Err(e) = icon.download_cached(invalidate_cache).await {
212                log::info!("{e:?}");
213            }
214        }
215
216        let urls: BTreeSet<_> = self.threads.iter().flat_map(|t| t.image_urls()).collect();
217        for url in urls {
218            if let Err(e) = download_cached_image(&url, invalidate_cache).await {
219                log::info!("{e:?}");
220            }
221        }
222    }
223}
224
225pub async fn download_cached_image(
226    url: &str,
227    invalidate_cache: bool,
228) -> Result<(Mime, Vec<u8>), Box<dyn Error>> {
229    fn image_cache_key(hash: &str, extension: &str) -> PathBuf {
230        format!("{CACHE_ROOT}/images/hash_{hash}.{extension}").into()
231    }
232
233    let hash = url_hash(url);
234
235    if !invalidate_cache {
236        if let Ok((mime, data)) = read_image_file(image_cache_key(&hash, "*")) {
237            return Ok((mime, data));
238        }
239    }
240
241    log::info!("Downloading image {hash} from {url}");
242
243    let (mime, data) = download_image(url).await?;
244
245    let mime = guess_image_mime(&data).unwrap_or(mime);
246
247    let extension = mime_to_image_extension(&mime).ok_or(format!("Invalid mime: {mime}"))?;
248
249    let cache_path = image_cache_key(&hash, &extension);
250    std::fs::create_dir_all(cache_path.parent().unwrap()).unwrap();
251    write_if_changed(cache_path, &data).unwrap();
252
253    Ok((mime, data))
254}
255
256async fn get_cached_glowfic<T>(
257    url: &str,
258    cache_path: &Path,
259    invalidate_cache: bool,
260) -> Result<Result<T, Vec<GlowficError>>, Box<dyn Error>>
261where
262    T: DeserializeOwned + Serialize,
263{
264    if !invalidate_cache {
265        if let Ok(data) = std::fs::read(cache_path) {
266            let parsed: Result<T, Vec<GlowficError>> = serde_json::from_slice(&data).unwrap();
267            if parsed.is_ok() {
268                return Ok(parsed);
269            }
270        }
271    }
272    let response = crate::api::get_glowfic(url).await?;
273
274    std::fs::create_dir_all(cache_path.parent().unwrap()).unwrap();
275    write_if_changed(cache_path, serde_json::to_vec_pretty(&response).unwrap()).unwrap();
276
277    Ok(response)
278}
279
280pub async fn download_image(url: &str) -> Result<(Mime, Vec<u8>), reqwest::Error> {
281    let response = http_client().get(url).send().await?;
282
283    let content_type = response.headers().get(CONTENT_TYPE).unwrap();
284    let mime = Mime::from_str(content_type.to_str().unwrap()).unwrap();
285
286    let data = response.bytes().await?;
287
288    Ok((mime, data.to_vec()))
289}
290fn read_image_file(path: PathBuf) -> Result<(Mime, Vec<u8>), Box<dyn Error>> {
291    let files: Vec<_> = glob::glob(path.to_str().unwrap()).unwrap().collect();
292
293    match &*files {
294        // If we find a single file, we are good to go.
295        [Ok(path)] => {
296            let data = std::fs::read(path).unwrap();
297
298            let extension = path.extension().unwrap().to_str().unwrap();
299            if let Some(mime) = extension_to_image_mime(extension) {
300                Ok((mime, data))
301            } else {
302                Err("Unsupprted extension in cached image.")?
303            }
304        }
305
306        // The way we changed the handling of icons with broken mimes could lead to
307        // multiple files for the same icon (but different extensions) being present.
308        // We delete and re-download them.
309        [_one, _two, _rest @ ..] => {
310            #[allow(clippy::manual_flatten)] // Flattening [Result]s hides errors.
311            for file in files {
312                if let Ok(file) = file {
313                    std::fs::remove_file(file).unwrap();
314                }
315            }
316
317            Err(format!("Found multiple files for image ({path:?}). Cleaning them up. No further action needed."))?
318        }
319
320        _ => Err("Did not find a match for image in the cache.")?,
321    }
322}
323
324/// Avoids updating the last-modified date of the file.
325pub fn write_if_changed(path: impl AsRef<Path>, contents: impl AsRef<[u8]>) -> std::io::Result<()> {
326    match std::fs::read(path.as_ref()) {
327        Ok(data) if data == contents.as_ref() => Ok(()),
328        Ok(_) | Err(_) => std::fs::write(path, contents),
329    }
330}
glowpub/cached.rs

glowpub/
cached.rs
