tiger_pkg/manager/
mod.rs

1pub mod lookup_cache;
2pub mod path_cache;
3
4use std::{
5    fmt::Display,
6    fs,
7    io::Cursor,
8    path::{Path, PathBuf},
9    str::FromStr,
10    sync::Arc,
11};
12
13use anyhow::Context;
14use binrw::{BinRead, BinReaderExt};
15use parking_lot::RwLock;
16use rayon::prelude::*;
17use rustc_hash::FxHashMap;
18use tracing::{debug_span, info, warn};
19
20use crate::{
21    d2_shared::PackageNamedTagEntry,
22    oodle,
23    package::{Package, PackagePlatform, UEntryHeader},
24    tag::TagHash64,
25    GameVersion, TagHash, Version,
26};
27
28#[derive(Clone, bincode::Decode, bincode::Encode)]
29pub struct HashTableEntryShort {
30    pub hash32: TagHash,
31    pub reference: TagHash,
32}
33
34#[derive(Default, bincode::Decode, bincode::Encode)]
35pub struct TagLookupIndex {
36    pub tag32_entries_by_pkg: FxHashMap<u16, Vec<UEntryHeader>>,
37    pub tag64_entries: FxHashMap<u64, HashTableEntryShort>,
38    pub tag32_to_tag64: FxHashMap<TagHash, TagHash64>,
39
40    pub named_tags: Vec<PackageNamedTagEntry>,
41}
42
43pub struct PackageManager {
44    pub package_dir: PathBuf,
45    pub package_paths: FxHashMap<u16, PackagePath>,
46    pub version: GameVersion,
47    pub platform: PackagePlatform,
48
49    /// Tag Lookup Index (TLI)
50    pub lookup: TagLookupIndex,
51
52    /// Packages that are currently open for reading
53    pkgs: RwLock<FxHashMap<u16, Arc<dyn Package>>>,
54}
55
56impl PackageManager {
57    pub fn new<P: AsRef<Path>>(
58        packages_dir: P,
59        version: GameVersion,
60        platform: Option<PackagePlatform>,
61    ) -> anyhow::Result<PackageManager> {
62        // All the latest packages
63        let mut packages: FxHashMap<u16, String> = Default::default();
64
65        let oo2core_3_path = packages_dir.as_ref().join("../bin/x64/oo2core_3_win64.dll");
66        let oo2core_9_path = packages_dir.as_ref().join("../bin/x64/oo2core_9_win64.dll");
67
68        if oo2core_3_path.exists() {
69            let mut o = oodle::OODLE_3.write();
70            if o.is_err() {
71                if let Ok(oodle) = oodle::Oodle::from_path(oo2core_3_path) {
72                    *o = Ok(oodle);
73                }
74            }
75        }
76
77        if oo2core_9_path.exists() {
78            let mut o = oodle::OODLE_9.write();
79            if o.is_err() {
80                if let Ok(oodle) = oodle::Oodle::from_path(oo2core_9_path) {
81                    *o = Ok(oodle);
82                }
83            }
84        }
85
86        let build_new_cache = match Self::validate_cache(version, platform, packages_dir.as_ref()) {
87            Ok(paths) => {
88                packages = paths;
89                false
90            }
91            Err(e) => {
92                warn!("Caches need to be rebuilt: {e}");
93                true
94            }
95        };
96
97        if build_new_cache {
98            info!("Creating new package cache for {}", version.id());
99            let path = packages_dir.as_ref();
100            // Every package in the given directory, including every patch
101            let mut packages_all = vec![];
102            debug_span!("Discover packages in directory").in_scope(|| -> anyhow::Result<()> {
103                for entry in fs::read_dir(path)? {
104                    let entry = entry?;
105                    let path = entry.path();
106                    if path.is_file() && path.to_string_lossy().to_lowercase().ends_with(".pkg") {
107                        packages_all.push(path.to_string_lossy().to_string());
108                    }
109                }
110
111                Ok(())
112            })?;
113
114            packages_all.sort_by_cached_key(|p| {
115                let p = PackagePath::parse_with_defaults(p);
116                (p.id, p.patch)
117            });
118
119            debug_span!("Filter latest packages").in_scope(|| {
120                for p in packages_all {
121                    let parts: Vec<&str> = p.split('_').collect();
122                    if let Some(Ok(pkg_id)) = parts
123                        .get(parts.len() - 2)
124                        .map(|s| u16::from_str_radix(s, 16))
125                    {
126                        packages.insert(pkg_id, p);
127                    } else {
128                        let _span = debug_span!("Open package to find package ID").entered();
129                        // Take the long route and extract the package ID from the header
130                        if let Ok(pkg) = version.open(&p) {
131                            if pkg.language().english_or_none() {
132                                packages.insert(pkg.pkg_id(), p);
133                            }
134                        }
135                    }
136                }
137            });
138        }
139
140        let package_paths: FxHashMap<u16, PackagePath> = packages
141            .into_iter()
142            .map(|(id, p)| (id, PackagePath::parse_with_defaults(&p)))
143            .collect();
144
145        let first_path = package_paths.values().next().context("No packages found")?;
146
147        let platform = if let Ok(pkg) = version.open(&first_path.path) {
148            pkg.platform()
149        } else {
150            PackagePlatform::from_str(first_path.platform.as_str())?
151        };
152
153        let mut s = Self {
154            package_dir: packages_dir.as_ref().to_path_buf(),
155            platform,
156            package_paths,
157            version,
158            lookup: Default::default(),
159            pkgs: Default::default(),
160        };
161
162        if build_new_cache {
163            s.build_lookup_tables();
164            s.write_package_cache().ok();
165            s.write_lookup_cache().ok();
166        } else if let Some(lookup_cache) = s.read_lookup_cache() {
167            s.lookup = lookup_cache;
168        } else {
169            info!("No valid index cache found, rebuilding");
170            s.build_lookup_tables();
171            s.write_lookup_cache().ok();
172        }
173
174        Ok(s)
175    }
176}
177
178impl PackageManager {
179    pub fn get_all_by_reference(&self, reference: u32) -> Vec<(TagHash, UEntryHeader)> {
180        self.lookup
181            .tag32_entries_by_pkg
182            .par_iter()
183            .map(|(p, e)| {
184                e.iter()
185                    .enumerate()
186                    .filter(|(_, e)| e.reference == reference)
187                    .map(|(i, e)| (TagHash::new(*p, i as _), e.clone()))
188                    .collect::<Vec<(TagHash, UEntryHeader)>>()
189            })
190            .flatten()
191            .collect()
192    }
193
194    pub fn get_all_by_type(&self, etype: u8, esubtype: Option<u8>) -> Vec<(TagHash, UEntryHeader)> {
195        self.lookup
196            .tag32_entries_by_pkg
197            .par_iter()
198            .map(|(p, e)| {
199                e.iter()
200                    .enumerate()
201                    .filter(|(_, e)| {
202                        e.file_type == etype
203                            && esubtype.map(|t| t == e.file_subtype).unwrap_or(true)
204                    })
205                    .map(|(i, e)| (TagHash::new(*p, i as _), e.clone()))
206                    .collect::<Vec<(TagHash, UEntryHeader)>>()
207            })
208            .flatten()
209            .collect()
210    }
211
212    fn get_or_load_pkg(&self, pkg_id: u16) -> anyhow::Result<Arc<dyn Package>> {
213        let _span = tracing::debug_span!("PackageManager::get_or_Load_pkg", pkg_id).entered();
214        let v = self.pkgs.read();
215        if let Some(pkg) = v.get(&pkg_id) {
216            Ok(Arc::clone(pkg))
217        } else {
218            drop(v);
219            let package_path = self
220                .package_paths
221                .get(&pkg_id)
222                .with_context(|| format!("Couldn't get a path for package id {pkg_id:04x}"))?;
223
224            let package = self
225                .version
226                .open(&package_path.path)
227                .with_context(|| format!("Failed to open package '{}'", package_path.filename))?;
228
229            self.pkgs.write().insert(pkg_id, Arc::clone(&package));
230            Ok(package)
231        }
232    }
233
234    pub fn read_tag(&self, tag: impl Into<TagHash>) -> anyhow::Result<Vec<u8>> {
235        let _span = tracing::debug_span!("PackageManager::read_tag").entered();
236        let tag = tag.into();
237        self.get_or_load_pkg(tag.pkg_id())?
238            .read_entry(tag.entry_index() as _)
239    }
240
241    pub fn read_tag64(&self, hash: impl Into<TagHash64>) -> anyhow::Result<Vec<u8>> {
242        let hash = hash.into();
243        let tag = self
244            .lookup
245            .tag64_entries
246            .get(&hash.0)
247            .context("Hash not found")?
248            .hash32;
249        self.read_tag(tag)
250    }
251
252    pub fn get_entry(&self, tag: impl Into<TagHash>) -> Option<UEntryHeader> {
253        let tag: TagHash = tag.into();
254
255        self.lookup
256            .tag32_entries_by_pkg
257            .get(&tag.pkg_id())?
258            .get(tag.entry_index() as usize)
259            .cloned()
260    }
261
262    pub fn get_named_tag(&self, name: &str, class_hash: u32) -> Option<TagHash> {
263        self.lookup
264            .named_tags
265            .iter()
266            .find(|n| n.name == name && n.class_hash == class_hash)
267            .map(|n| n.hash)
268    }
269
270    pub fn get_named_tags_by_class(&self, class_hash: u32) -> Vec<(String, TagHash)> {
271        self.lookup
272            .named_tags
273            .iter()
274            .filter(|n| n.class_hash == class_hash)
275            .map(|n| (n.name.clone(), n.hash))
276            .collect()
277    }
278
279    /// Find the name of a tag by its hash, if it has one.
280    pub fn get_tag_name(&self, tag: impl Into<TagHash>) -> Option<String> {
281        let tag: TagHash = tag.into();
282        self.lookup
283            .named_tags
284            .iter()
285            .find(|n| n.hash == tag)
286            .map(|n| n.name.clone())
287    }
288
289    pub fn get_tag64_for_tag32(&self, tag: impl Into<TagHash>) -> Option<TagHash64> {
290        let tag: TagHash = tag.into();
291        self.lookup.tag32_to_tag64.get(&tag).copied()
292    }
293
294    /// Read any BinRead type
295    pub fn read_tag_binrw<'a, T: BinRead>(&self, tag: impl Into<TagHash>) -> anyhow::Result<T>
296    where
297        T::Args<'a>: Default + Clone,
298    {
299        let tag = tag.into();
300        let data = self.read_tag(tag)?;
301        let mut cursor = Cursor::new(&data);
302        Ok(cursor.read_type(self.version.endian())?)
303    }
304
305    /// Read any BinRead type
306    pub fn read_tag64_binrw<'a, T: BinRead>(&self, hash: impl Into<TagHash64>) -> anyhow::Result<T>
307    where
308        T::Args<'a>: Default + Clone,
309    {
310        let data = self.read_tag64(hash)?;
311        let mut cursor = Cursor::new(&data);
312        Ok(cursor.read_type(self.version.endian())?)
313    }
314}
315
316#[derive(Debug, Clone)]
317pub struct PackagePath {
318    /// eg. ps3, w64
319    pub platform: String,
320    /// eg. arch_fallen, dungeon_prophecy, europa
321    pub name: String,
322
323    /// 2-letter language code (en, fr, de, etc.)
324    pub language: Option<String>,
325
326    /// eg. 0059, 043c, unp1, unp2
327    pub id: String,
328    pub patch: u8,
329
330    /// Full path to the package
331    pub path: String,
332    pub filename: String,
333}
334
335impl PackagePath {
336    /// Example path: ps3_arch_fallen_0059_0.pkg
337    pub fn parse(path: &str) -> Option<Self> {
338        let path_filename = Path::new(path).file_name()?.to_string_lossy();
339        let parts: Vec<&str> = path_filename.split('_').collect();
340        if parts.len() < 4 {
341            return None;
342        }
343
344        let platform = parts[0].to_string();
345        let mut name = parts[1..parts.len() - 2].join("_");
346        let mut id = parts[parts.len() - 2].to_string();
347        let mut language = None;
348        if id.len() == 2 {
349            // ID is actually language code
350            language = Some(id.clone());
351            name = parts[1..parts.len() - 3].join("_");
352            id = parts[parts.len() - 3].to_string();
353        }
354
355        let patch = parts[parts.len() - 1].split('.').next()?.parse().ok()?;
356
357        Some(Self {
358            platform,
359            name,
360            language,
361            id,
362            patch,
363            path: path.to_string(),
364            filename: path_filename.to_string(),
365        })
366    }
367
368    pub fn parse_with_defaults(path: &str) -> Self {
369        let path_filename = Path::new(path)
370            .file_name()
371            .map_or(path.to_string(), |p| p.to_string_lossy().to_string());
372        Self::parse(path).unwrap_or_else(|| Self {
373            platform: "unknown".to_string(),
374            name: "unknown".to_string(),
375            id: "unknown".to_string(),
376            language: None,
377            patch: 0,
378            path: path.to_string(),
379            filename: path_filename,
380        })
381    }
382}
383
384impl Display for PackagePath {
385    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
386        write!(f, "{}", self.filename)
387    }
388}
tiger_pkg/manager/mod.rs

tiger_pkg/manager/
mod.rs
