oxygengine_core/assets/
database.rs

1use crate::{
2    assets::{
3        asset::{Asset, AssetId},
4        protocol::{AssetLoadResult, AssetProtocol, AssetVariant, Meta},
5    },
6    fetch::{FetchEngine, FetchProcess, FetchStatus},
7};
8use std::{any::TypeId, collections::HashMap};
9
10#[derive(Debug, Clone, PartialEq)]
11pub enum LoadStatus {
12    InvalidPath(String),
13    UnknownProtocol(String),
14    FetchError(FetchStatus),
15    NoFetchEngine,
16}
17
18pub trait AssetsDatabaseErrorReporter: Send + Sync {
19    fn on_report(&mut self, protocol: &str, path: &str, message: &str);
20}
21
22#[derive(Debug, Default, Copy, Clone)]
23pub struct LoggerAssetsDatabaseErrorReporter;
24
25impl AssetsDatabaseErrorReporter for LoggerAssetsDatabaseErrorReporter {
26    fn on_report(&mut self, protocol: &str, path: &str, message: &str) {
27        error!(
28            "Assets database loading `{}://{}` error: {}",
29            protocol, path, message
30        );
31    }
32}
33
34pub struct AssetsDatabase {
35    pub max_bytes_per_frame: Option<usize>,
36    fetch_engines: Vec<Box<dyn FetchEngine>>,
37    protocols: HashMap<String, Box<dyn AssetProtocol>>,
38    assets: HashMap<AssetId, (String, Asset)>,
39    table: HashMap<String, AssetId>,
40    loading: HashMap<String, (String, Box<FetchProcess>)>,
41    #[allow(clippy::type_complexity)]
42    yielded: HashMap<String, (String, Meta, Vec<(String, String)>)>,
43    lately_loaded: Vec<(String, AssetId)>,
44    lately_unloaded: Vec<(String, AssetId)>,
45    error_reporters: HashMap<TypeId, Box<dyn AssetsDatabaseErrorReporter>>,
46    defer_lately_cleanup: bool,
47}
48
49impl AssetsDatabase {
50    pub fn new<FE>(fetch_engine: FE) -> Self
51    where
52        FE: FetchEngine + 'static,
53    {
54        Self {
55            max_bytes_per_frame: None,
56            fetch_engines: vec![Box::new(fetch_engine)],
57            protocols: Default::default(),
58            assets: Default::default(),
59            table: Default::default(),
60            loading: Default::default(),
61            yielded: Default::default(),
62            lately_loaded: vec![],
63            lately_unloaded: vec![],
64            error_reporters: Default::default(),
65            defer_lately_cleanup: true,
66        }
67    }
68
69    pub fn register_error_reporter<T>(&mut self, reporter: T)
70    where
71        T: AssetsDatabaseErrorReporter + 'static,
72    {
73        self.error_reporters
74            .insert(TypeId::of::<T>(), Box::new(reporter));
75    }
76
77    pub fn unregister_error_reporter<T>(&mut self)
78    where
79        T: AssetsDatabaseErrorReporter + 'static,
80    {
81        self.error_reporters.remove(&TypeId::of::<T>());
82    }
83
84    pub fn loaded_count(&self) -> usize {
85        self.assets.len()
86    }
87
88    pub fn loaded_paths(&self) -> Vec<String> {
89        self.assets
90            .iter()
91            .map(|(_, (_, a))| a.to_full_path())
92            .collect()
93    }
94
95    pub fn loaded_ids(&self) -> Vec<AssetId> {
96        self.assets.keys().copied().collect()
97    }
98
99    pub fn loading_count(&self) -> usize {
100        self.loading.len()
101    }
102
103    pub fn loading_paths(&self) -> Vec<String> {
104        let mut result = self
105            .loading
106            .iter()
107            .map(|(path, (prot, _))| format!("{}://{}", prot, path))
108            .collect::<Vec<_>>();
109        result.sort();
110        result
111    }
112
113    pub fn yielded_count(&self) -> usize {
114        self.yielded.len()
115    }
116
117    pub fn yielded_paths(&self) -> Vec<String> {
118        let mut result = self
119            .yielded
120            .iter()
121            .map(|(path, (prot, _, _))| format!("{}://{}", prot, path))
122            .collect::<Vec<_>>();
123        result.sort();
124        result
125    }
126
127    pub fn yielded_deps_count(&self) -> usize {
128        self.yielded
129            .iter()
130            .map(|(_, (_, _, list))| list.len())
131            .sum()
132    }
133
134    pub fn yielded_deps_paths(&self) -> Vec<String> {
135        let mut result = self
136            .yielded
137            .iter()
138            .flat_map(|(_, (_, _, list))| {
139                list.iter().map(|(p, _)| p.to_owned()).collect::<Vec<_>>()
140            })
141            .collect::<Vec<_>>();
142        result.sort();
143        result.dedup();
144        result
145    }
146
147    pub fn lately_loaded(&self) -> impl Iterator<Item = &AssetId> {
148        self.lately_loaded.iter().map(|(_, id)| id)
149    }
150
151    pub fn lately_loaded_paths(&self) -> impl Iterator<Item = &str> {
152        self.lately_loaded.iter().map(|(path, _)| path.as_str())
153    }
154
155    pub fn lately_loaded_protocol<'a>(
156        &'a self,
157        protocol: &'a str,
158    ) -> impl Iterator<Item = &'a AssetId> {
159        self.lately_loaded
160            .iter()
161            .filter_map(move |(prot, id)| if protocol == prot { Some(id) } else { None })
162    }
163
164    pub fn lately_unloaded(&self) -> impl Iterator<Item = &AssetId> {
165        self.lately_unloaded.iter().map(|(_, id)| id)
166    }
167
168    pub fn lately_unloaded_paths(&self) -> impl Iterator<Item = &str> {
169        self.lately_unloaded.iter().map(|(path, _)| path.as_str())
170    }
171
172    pub fn lately_unloaded_protocol<'a>(
173        &'a self,
174        protocol: &'a str,
175    ) -> impl Iterator<Item = &'a AssetId> {
176        self.lately_unloaded
177            .iter()
178            .filter_map(move |(prot, id)| if protocol == prot { Some(id) } else { None })
179    }
180
181    pub fn is_ready(&self) -> bool {
182        self.loading.is_empty() && self.yielded.is_empty()
183    }
184
185    pub fn are_ready<I, S>(&self, iter: I) -> bool
186    where
187        I: IntoIterator<Item = S>,
188        S: AsRef<str>,
189    {
190        iter.into_iter().all(|path| {
191            let path = Self::clean_path(path.as_ref());
192            self.table.contains_key(path)
193                && !self.loading.contains_key(path)
194                && !self.yielded.contains_key(path)
195        })
196    }
197
198    pub fn has_fetch_engine(&self) -> bool {
199        !self.fetch_engines.is_empty()
200    }
201
202    pub fn fetch_engines_stack_size(&self) -> usize {
203        self.fetch_engines.len()
204    }
205
206    pub fn push_fetch_engine(&mut self, fetch_engine: Box<dyn FetchEngine>) {
207        self.fetch_engines.push(fetch_engine);
208    }
209
210    pub fn pop_fetch_engine(&mut self) -> Option<Box<dyn FetchEngine>> {
211        self.fetch_engines.pop()
212    }
213
214    pub fn fetch_engine(&self) -> Option<&dyn FetchEngine> {
215        self.fetch_engines.last().map(|engine| engine.as_ref())
216    }
217
218    pub fn fetch_engine_mut(&mut self) -> Option<&mut (dyn FetchEngine + 'static)> {
219        self.fetch_engines.last_mut().map(|engine| engine.as_mut())
220    }
221
222    pub fn with_fetch_engine<F, R>(&mut self, mut action: F) -> Option<R>
223    where
224        F: FnMut(&mut dyn FetchEngine) -> R,
225    {
226        Some(action(self.fetch_engine_mut()?))
227    }
228
229    pub fn register<FE>(&mut self, mut protocol: FE)
230    where
231        FE: AssetProtocol + 'static,
232    {
233        protocol.on_register();
234        let name = protocol.name().to_owned();
235        self.protocols.insert(name, Box::new(protocol));
236    }
237
238    pub fn unregister(&mut self, protocol_name: &str) -> Option<Box<dyn AssetProtocol>> {
239        if let Some(mut protocol) = self.protocols.remove(protocol_name) {
240            protocol.on_unregister();
241            Some(protocol)
242        } else {
243            None
244        }
245    }
246
247    pub fn load(&mut self, path: &str) -> Result<(), LoadStatus> {
248        if self.table.contains_key(path) {
249            return Ok(());
250        }
251        let path = Self::clean_path(path);
252        let parts = path.split("://").take(2).collect::<Vec<_>>();
253        if parts.len() == 2 {
254            let prot = parts[0];
255            let subpath = parts[1];
256            if self.protocols.contains_key(prot) {
257                if let Some(engine) = self.fetch_engine_mut() {
258                    let reader = engine.fetch(subpath);
259                    match reader {
260                        Ok(reader) => {
261                            self.loading
262                                .insert(subpath.to_owned(), (prot.to_owned(), reader));
263                            Ok(())
264                        }
265                        Err(status) => Err(LoadStatus::FetchError(status)),
266                    }
267                } else {
268                    Err(LoadStatus::NoFetchEngine)
269                }
270            } else {
271                Err(LoadStatus::UnknownProtocol(prot.to_owned()))
272            }
273        } else {
274            Err(LoadStatus::InvalidPath(path.to_owned()))
275        }
276    }
277
278    pub fn insert(&mut self, asset: Asset) -> AssetId {
279        let path = asset.to_full_path();
280        let path = Self::clean_path(&path);
281        let id = asset.id();
282        self.lately_loaded.push((asset.protocol().to_owned(), id));
283        self.assets.insert(id, (path.to_owned(), asset));
284        self.table.insert(path.to_owned(), id);
285        id
286    }
287
288    pub fn remove_by_id(&mut self, id: AssetId) -> Option<Asset> {
289        if let Some((path, asset)) = self.assets.remove(&id) {
290            self.table.remove(&path);
291            self.lately_unloaded.push((asset.protocol().to_owned(), id));
292            if let Some(protocol) = self.protocols.get_mut(asset.protocol()) {
293                if let Some(list) = protocol.on_unload(&asset) {
294                    self.remove_by_variants(&list);
295                }
296            }
297            Some(asset)
298        } else {
299            None
300        }
301    }
302
303    pub fn remove_by_path(&mut self, path: &str) -> Option<Asset> {
304        let path = Self::clean_path(path);
305        if let Some(id) = self.table.remove(path) {
306            if let Some((_, asset)) = self.assets.remove(&id) {
307                self.lately_unloaded.push((asset.protocol().to_owned(), id));
308                if let Some(protocol) = self.protocols.get_mut(asset.protocol()) {
309                    if let Some(list) = protocol.on_unload(&asset) {
310                        self.remove_by_variants(&list);
311                    }
312                }
313                return Some(asset);
314            }
315        }
316        None
317    }
318
319    pub fn remove_by_variants(&mut self, variants: &[AssetVariant]) {
320        for v in variants {
321            match v {
322                AssetVariant::Id(id) => self.remove_by_id(*id),
323                AssetVariant::Path(path) => self.remove_by_path(path),
324            };
325        }
326    }
327
328    pub fn id_by_path(&self, path: &str) -> Option<AssetId> {
329        let path = Self::clean_path(path);
330        self.table.get(path).cloned()
331    }
332
333    pub fn path_by_id(&self, id: AssetId) -> Option<&str> {
334        self.assets.get(&id).map(|(path, _)| path.as_str())
335    }
336
337    pub fn asset_by_id(&self, id: AssetId) -> Option<&Asset> {
338        self.assets.get(&id).map(|(_, asset)| asset)
339    }
340
341    pub fn asset_by_path(&self, path: &str) -> Option<&Asset> {
342        let path = Self::clean_path(path);
343        if let Some(id) = self.table.get(path) {
344            if let Some((_, asset)) = self.assets.get(id) {
345                return Some(asset);
346            }
347        }
348        None
349    }
350
351    pub fn defer_lately_cleanup(&mut self) {
352        self.defer_lately_cleanup = true;
353    }
354
355    pub fn process(&mut self) {
356        if self.defer_lately_cleanup {
357            self.defer_lately_cleanup = false;
358        } else {
359            self.lately_loaded.clear();
360            self.lately_unloaded.clear();
361        }
362        let to_dispatch = {
363            let mut bytes_read = 0;
364            self.loading
365                .iter()
366                .filter_map(|(path, (prot, reader))| {
367                    if bytes_read < self.max_bytes_per_frame.unwrap_or(std::usize::MAX) {
368                        if let Some(data) = reader.read() {
369                            bytes_read += data.len();
370                            return Some((path.to_owned(), prot.to_owned(), data));
371                        }
372                    }
373                    None
374                })
375                .collect::<Vec<_>>()
376        };
377        for (path, prot, data) in to_dispatch {
378            if let Some(protocol) = self.protocols.get_mut(&prot) {
379                match protocol.on_load_with_path(&path, data) {
380                    AssetLoadResult::Data(data) => {
381                        let asset = Asset::new_boxed(&prot, &path, data);
382                        self.insert(asset);
383                    }
384                    AssetLoadResult::Yield(meta, list) => {
385                        let list = list
386                            .into_iter()
387                            .filter(|(_, path)| self.load(path).is_ok())
388                            .collect();
389                        self.yielded.insert(path, (prot, meta, list));
390                    }
391                    AssetLoadResult::Error(message) => {
392                        for reporter in self.error_reporters.values_mut() {
393                            reporter.on_report(&prot, &path, &message);
394                        }
395                    }
396                }
397            }
398        }
399        self.loading.retain(|_, (_, reader)| {
400            matches!(
401                reader.status(),
402                FetchStatus::InProgress(_) | FetchStatus::Done
403            )
404        });
405        let yielded = std::mem::take(&mut self.yielded);
406        for (path, (prot, meta, list)) in yielded {
407            if list.iter().all(|(_, path)| self.table.contains_key(path)) {
408                let ptr = self as *const Self;
409                if let Some(protocol) = self.protocols.get_mut(&prot) {
410                    let list = list
411                        .iter()
412                        .map(|(key, path)| unsafe {
413                            let asset = &(*ptr).table[path];
414                            let asset = &(*ptr).assets[asset].1;
415                            (key.as_str(), asset)
416                        })
417                        .collect::<Vec<_>>();
418                    match protocol.on_resume(meta, &list) {
419                        AssetLoadResult::Data(data) => {
420                            let asset = Asset::new_boxed(&prot, &path, data);
421                            self.insert(asset);
422                        }
423                        AssetLoadResult::Yield(meta, list) => {
424                            let list = list
425                                .into_iter()
426                                .filter(|(_, path)| self.load(path).is_ok())
427                                .collect();
428                            self.yielded.insert(path, (prot, meta, list));
429                        }
430                        AssetLoadResult::Error(message) => {
431                            for reporter in self.error_reporters.values_mut() {
432                                reporter.on_report(&prot, &path, &message);
433                            }
434                        }
435                    }
436                }
437            } else {
438                self.yielded.insert(path, (prot, meta, list));
439            }
440        }
441    }
442
443    fn clean_path(path: &str) -> &str {
444        path.strip_prefix('*').unwrap_or(path)
445    }
446}
447
448#[cfg(test)]
449mod tests {
450    use super::*;
451    use crate::{
452        assets::protocols::{
453            meta::{MetaAsset, MetaAssetProtocol},
454            text::{TextAsset, TextAssetProtocol},
455        },
456        fetch::*,
457    };
458
459    #[test]
460    fn test_database() {
461        let list = serde_json::to_string(
462            &MetaAsset::default()
463                .with_target("txt://a.txt")
464                .with_target("txt://b.txt"),
465        )
466        .unwrap();
467        let mut fetch_engine = engines::map::MapFetchEngine::default();
468        fetch_engine
469            .map
470            .insert("assets.asset".to_owned(), list.into_bytes().to_vec());
471        fetch_engine.map.insert("a.txt".to_owned(), b"A".to_vec());
472        fetch_engine.map.insert("b.txt".to_owned(), b"B".to_vec());
473
474        let mut database = AssetsDatabase::new(fetch_engine);
475        database.register(TextAssetProtocol);
476        database.register(MetaAssetProtocol);
477        assert_eq!(database.load("meta://assets.asset"), Ok(()));
478        assert_eq!(database.loaded_count(), 0);
479        assert_eq!(database.loading_count(), 1);
480        assert_eq!(database.yielded_count(), 0);
481        assert_eq!(database.yielded_deps_count(), 0);
482
483        for _ in 0..2 {
484            database.process();
485        }
486        assert_eq!(database.loaded_count(), 3);
487        assert_eq!(database.loading_count(), 0);
488        assert_eq!(database.yielded_count(), 0);
489        assert_eq!(database.yielded_deps_count(), 0);
490
491        assert!(database.asset_by_path("meta://assets.asset").is_some());
492        assert_eq!(
493            &database
494                .asset_by_path("meta://assets.asset")
495                .unwrap()
496                .to_full_path(),
497            "meta://assets.asset"
498        );
499        assert!(database
500            .asset_by_path("meta://assets.asset")
501            .unwrap()
502            .is::<MetaAsset>());
503        assert_eq!(
504            database
505                .asset_by_path("meta://assets.asset")
506                .unwrap()
507                .get::<MetaAsset>()
508                .unwrap()
509                .target()
510                .collect::<Vec<_>>(),
511            vec!["txt://a.txt", "txt://b.txt"],
512        );
513
514        assert!(database.asset_by_path("txt://a.txt").is_some());
515        assert!(database
516            .asset_by_path("txt://a.txt")
517            .unwrap()
518            .is::<TextAsset>());
519        assert_eq!(
520            database
521                .asset_by_path("txt://a.txt")
522                .unwrap()
523                .get::<TextAsset>()
524                .unwrap()
525                .get(),
526            "A"
527        );
528
529        assert!(database.asset_by_path("txt://b.txt").is_some());
530        assert_eq!(
531            database
532                .asset_by_path("txt://b.txt")
533                .unwrap()
534                .get::<TextAsset>()
535                .unwrap()
536                .get(),
537            "B"
538        );
539
540        assert!(database.remove_by_path("meta://assets.asset").is_some());
541        assert_eq!(database.loaded_count(), 0);
542        assert_eq!(database.loading_count(), 0);
543        assert_eq!(database.yielded_count(), 0);
544        assert_eq!(database.yielded_deps_count(), 0);
545    }
546}