runefs/
index.rs

1use std::{
2    collections::{hash_map, HashMap},
3    fs::File,
4    io::Read,
5    path::Path,
6    slice::Iter,
7};
8
9#[cfg(feature = "serde")]
10use serde::{Deserialize, Serialize};
11
12use crate::{
13    archive::{ArchiveMetadata, ArchiveRef, ARCHIVE_REF_LEN},
14    error::{ParseError, ReadError},
15    Dat2, REFERENCE_TABLE_ID,
16};
17use itertools::izip;
18use nom::{
19    bytes::complete::take,
20    combinator::cond,
21    multi::{many0, many_m_n},
22    number::complete::{be_i32, be_u16, be_u32, be_u8},
23};
24
25use crate::codec::{Buffer, Decoded};
26use crate::parse::be_u32_smart;
27
28pub const IDX_PREFIX: &str = "main_file_cache.idx";
29
30/// A list of valid indices.
31#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
32#[derive(Clone, Debug, Default)]
33pub struct Indices(pub(crate) HashMap<u8, Index>);
34
35impl Indices {
36    /// Allocates an `Index` for every valid index file in the cache directory.
37    ///
38    /// An index is considered _valid_ if it is present, meaning it will scan the directory
39    /// for the `.idx#` suffix and load them into memory.
40    ///
41    /// # Errors
42    ///
43    /// Constructing this type is quite error prone, it needs to do quite a bit of book-keeping
44    /// to get its allocation right. However, if the cache is unchanged _and_ in its proper format
45    /// it will, most likely, succeed.
46    ///
47    /// The primary errors have to do with I/O, in order to read every index successfully it needs
48    /// a `Dat2` reference and the metadata index.
49    ///
50    /// If an index is found it needs to load its entire contents and parse it, failure at this point
51    /// is considered a bug.
52    pub fn new<P: AsRef<Path>>(path: P) -> crate::Result<Self> {
53        let path = path.as_ref();
54
55        let ref_index = Index::from_path(
56            REFERENCE_TABLE_ID,
57            path.join(format!("{}{}", IDX_PREFIX, REFERENCE_TABLE_ID)),
58        )?;
59        let dat2 = Dat2::new(path.join(crate::MAIN_DATA))?;
60        let mut indices = HashMap::with_capacity(255);
61
62        for p in std::fs::read_dir(path)? {
63            let path = p?.path();
64
65            if let Some(ext) = path.extension().and_then(std::ffi::OsStr::to_str) {
66                if let Some(index_id) = ext.strip_prefix("idx") {
67                    let index_id: u8 = index_id.parse().expect("invalid extension format");
68                    if index_id == 255 {
69                        continue;
70                    }
71                    let mut index = Index::from_path(index_id, path)?;
72                    let archive_ref = ref_index.archive_refs.get(&(index_id as u32)).ok_or(
73                        ReadError::ArchiveNotFound {
74                            idx: REFERENCE_TABLE_ID,
75                            arc: index_id as u32,
76                        },
77                    )?;
78                    if archive_ref.length != 0 {
79                        index.metadata = dat2.metadata(archive_ref)?;
80                    }
81                    indices.insert(index_id, index);
82                }
83            }
84        }
85
86        indices.insert(REFERENCE_TABLE_ID, ref_index);
87
88        Ok(Self(indices))
89    }
90
91    pub fn get(&self, key: &u8) -> Option<&Index> {
92        self.0.get(key)
93    }
94
95    pub fn count(&self) -> usize {
96        self.0.len()
97    }
98}
99
100/// A virtual file type for every `.idx` in the cache directory.
101#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
102#[derive(Clone, Debug, Default)]
103pub struct Index {
104    pub id: u8,
105    pub archive_refs: HashMap<u32, ArchiveRef>,
106    pub metadata: IndexMetadata,
107}
108
109impl Index {
110    /// Creates an `Index` from a file path.
111    ///
112    /// # Panics
113    ///
114    /// When an index is loaded the given id and its file extension are compared, if these mismatch
115    /// it is considered a bug.
116    ///
117    /// # Errors
118    ///
119    /// The primary errors concern I/O where the file couldn't be opened or read.
120    pub fn from_path<P: AsRef<Path>>(id: u8, path: P) -> crate::Result<Self> {
121        let path = path.as_ref();
122        let index_extension = format!("idx{}", id);
123        let extension = path
124            .extension()
125            .and_then(std::ffi::OsStr::to_str)
126            .unwrap_or("nothing");
127
128        if extension != index_extension {
129            panic!("index extension mismatch: expected {index_extension} but found {extension}");
130        }
131
132        let mut file = File::open(path)?;
133        let mut buffer = Vec::with_capacity(file.metadata()?.len() as usize);
134        file.read_to_end(&mut buffer)?;
135
136        Self::from_buffer(id, &buffer)
137    }
138
139    pub(crate) fn from_buffer(id: u8, buffer: &[u8]) -> crate::Result<Self> {
140        let mut archive_refs = HashMap::new();
141
142        for (archive_id, archive_data) in buffer.chunks_exact(ARCHIVE_REF_LEN).enumerate() {
143            let archive_id = archive_id as u32;
144
145            let archive_ref = match ArchiveRef::from_buffer(archive_id, id, archive_data) {
146                Ok(archive) => archive,
147                Err(_) => return Err(ParseError::Archive(archive_id).into()),
148            };
149            archive_refs.insert(archive_id, archive_ref);
150        }
151
152        Ok(Self {
153            id,
154            archive_refs,
155            metadata: IndexMetadata::default(),
156        })
157    }
158}
159
160impl IntoIterator for Indices {
161    type Item = (u8, Index);
162    type IntoIter = hash_map::IntoIter<u8, Index>;
163
164    #[inline]
165    fn into_iter(self) -> Self::IntoIter {
166        self.0.into_iter()
167    }
168}
169
170impl<'a> IntoIterator for &'a Indices {
171    type Item = (&'a u8, &'a Index);
172    type IntoIter = hash_map::Iter<'a, u8, Index>;
173
174    #[inline]
175    fn into_iter(self) -> Self::IntoIter {
176        self.0.iter()
177    }
178}
179
180/// All of the index metadata fetched through `Dat2` from the metadata table.
181#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
182#[derive(Default, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]
183pub struct IndexMetadata(Vec<ArchiveMetadata>);
184
185impl IndexMetadata {
186    /// Takes a specific raw metadata buffer and turns it into a `IndexMetadata`. 
187    /// 
188    /// # Errors
189    /// 
190    /// If, for what ever reason, the buffer does not _exactly_ adhere to the correct
191    /// format parsing will fail.
192    pub fn from_buffer(buffer: Buffer<Decoded>) -> crate::Result<Self> {
193        Self::from_slice(buffer.as_slice())
194    }
195
196    pub(crate) fn from_slice(buffer: &[u8]) -> crate::Result<Self> {
197        let (buffer, protocol) = be_u8(buffer)?;
198        // TODO: should actually parse this and add it to the struct
199        let (buffer, _) = cond(protocol >= 6, be_u32)(buffer)?;
200        let (buffer, identified, whirlpool, codec, hash) = parse_identified(buffer)?;
201        let (buffer, archive_count) = parse_archive_count(buffer, protocol)?;
202        let (buffer, ids) = parse_ids(buffer, protocol, archive_count)?;
203        let (buffer, name_hashes) = parse_hashes(buffer, identified, archive_count)?;
204        let (buffer, crcs) = many_m_n(0, archive_count, be_u32)(buffer)?;
205        let (buffer, hashes) = parse_hashes(buffer, hash, archive_count)?;
206        let (buffer, whirlpools) = parse_whirlpools(buffer, whirlpool, archive_count)?;
207        // skip for now TODO: should also be saved in the struct
208        //let (buffer, compressed, decompressed) = parse_codec(buffer, codec, archive_count)?;
209        let (buffer, _) = cond(codec, many_m_n(0, archive_count * 8, be_u8))(buffer)?;
210        let (buffer, versions) = many_m_n(0, archive_count, be_u32)(buffer)?;
211        let (buffer, entry_counts) = parse_entry_counts(buffer, protocol, archive_count)?;
212        let (_, valid_ids) = parse_valid_ids(buffer, protocol, &entry_counts)?;
213        let mut archives = Vec::with_capacity(archive_count);
214        let mut last_archive_id = 0;
215        let archive_data = izip!(
216            ids,
217            name_hashes,
218            crcs,
219            hashes,
220            whirlpools,
221            versions,
222            entry_counts,
223            valid_ids
224        );
225        for (id, name_hash, crc, hash, whirlpool, version, entry_count, valid_ids) in archive_data {
226            last_archive_id += id as i32;
227
228            archives.push(ArchiveMetadata {
229                id: last_archive_id as u32,
230                name_hash,
231                crc,
232                hash,
233                whirlpool,
234                version,
235                entry_count,
236                valid_ids,
237            });
238        }
239        Ok(Self(archives))
240    }
241
242    #[inline]
243    pub fn iter(&self) -> Iter<'_, ArchiveMetadata> {
244        self.0.iter()
245    }
246}
247
248impl std::ops::Index<usize> for IndexMetadata {
249    type Output = ArchiveMetadata;
250
251    fn index(&self, index: usize) -> &Self::Output {
252        &self.0[index]
253    }
254}
255
256impl IntoIterator for IndexMetadata {
257    type Item = ArchiveMetadata;
258    type IntoIter = std::vec::IntoIter<ArchiveMetadata>;
259
260    #[inline]
261    fn into_iter(self) -> Self::IntoIter {
262        self.0.into_iter()
263    }
264}
265
266impl<'a> IntoIterator for &'a IndexMetadata {
267    type Item = &'a ArchiveMetadata;
268    type IntoIter = Iter<'a, ArchiveMetadata>;
269
270    #[inline]
271    fn into_iter(self) -> Self::IntoIter {
272        self.0.iter()
273    }
274}
275
276fn parse_identified(buffer: &[u8]) -> crate::Result<(&[u8], bool, bool, bool, bool)> {
277    let (buffer, identified) = be_u8(buffer)?;
278
279    let whirlpool = (2 & identified) != 0;
280    let codec = (identified & 4) != 0;
281    let hash = (identified & 8) != 0;
282    let identified = (1 & identified) != 0;
283
284    Ok((buffer, identified, whirlpool, codec, hash))
285}
286
287fn parse_hashes(
288    buffer: &[u8],
289    hash: bool,
290    archive_count: usize,
291) -> crate::Result<(&[u8], Vec<i32>)> {
292    let (buffer, taken) = cond(hash, take(archive_count * 4))(buffer)?;
293    let (_, mut hashes) = many0(be_i32)(taken.unwrap_or(&[]))?;
294
295    if hashes.len() != archive_count {
296        hashes = vec![0; archive_count * 4];
297    }
298
299    Ok((buffer, hashes))
300}
301
302fn parse_whirlpools(
303    buffer: &[u8],
304    whirlpool: bool,
305    archive_count: usize,
306) -> crate::Result<(&[u8], Vec<[u8; 64]>)> {
307    let (buffer, taken) = cond(whirlpool, take(archive_count * 64))(buffer)?;
308    let mut whirlpools = vec![[0; 64]; archive_count];
309
310    for (index, chunk) in taken.unwrap_or(&[]).chunks_exact(64).enumerate() {
311        whirlpools[index].copy_from_slice(chunk);
312    }
313    if whirlpools.len() != archive_count {
314        whirlpools = vec![[0; 64]; archive_count];
315    }
316
317    Ok((buffer, whirlpools))
318}
319
320// fn parse_codec(buffer: &[u8], codec: bool, archive_count: usize) -> crate::Result<(&[u8], Vec<u32>, Vec<u32>)> {
321//     todo!()
322// }
323
324fn parse_valid_ids<'a>(
325    mut buffer: &'a [u8],
326    protocol: u8,
327    entry_counts: &[usize],
328) -> crate::Result<(&'a [u8], Vec<Vec<u32>>)> {
329    let mut result = Vec::with_capacity(entry_counts.len());
330
331    for entry_count in entry_counts {
332        let (buf, id_modifiers) = if protocol >= 7 {
333            many_m_n(0, *entry_count, be_u32_smart)(buffer)?
334        } else {
335            let (buf, result) = many_m_n(0, *entry_count, be_u16)(buffer)?;
336            let result = result.iter().map(|&id_mod| id_mod as u32).collect();
337
338            (buf, result)
339        };
340        buffer = buf;
341
342        let mut ids = Vec::with_capacity(id_modifiers.len());
343        let mut id = 0_u32;
344        for current_id in id_modifiers {
345            id += current_id;
346            ids.push(id);
347        }
348
349        result.push(ids);
350    }
351
352    Ok((buffer, result))
353}
354
355fn parse_archive_count(buffer: &[u8], protocol: u8) -> crate::Result<(&[u8], usize)> {
356    let (buffer, value) = if protocol >= 7 {
357        be_u32_smart(buffer)?
358    } else {
359        let (buf, res) = be_u16(buffer)?;
360        (buf, res as u32)
361    };
362
363    Ok((buffer, value as usize))
364}
365
366fn parse_ids(
367    buffer: &[u8],
368    protocol: u8,
369    archive_count: usize,
370) -> crate::Result<(&[u8], Vec<u32>)> {
371    let (buffer, ids) = if protocol >= 7 {
372        many_m_n(0, archive_count, be_u32_smart)(buffer)?
373    } else {
374        let (buf, res) = many_m_n(0, archive_count, be_u16)(buffer)?;
375        let res = res.iter().map(|&ec| ec as u32).collect();
376        (buf, res)
377    };
378
379    Ok((buffer, ids))
380}
381
382fn parse_entry_counts(
383    buffer: &[u8],
384    protocol: u8,
385    archive_count: usize,
386) -> crate::Result<(&[u8], Vec<usize>)> {
387    let (buffer, entry_counts) = if protocol >= 7 {
388        many_m_n(0, archive_count, be_u32_smart)(buffer)?
389    } else {
390        let (buf, res) = many_m_n(0, archive_count, be_u16)(buffer)?;
391        let res = res.iter().map(|&ec| ec as u32).collect();
392
393        (buf, res)
394    };
395
396    let entry_counts: Vec<usize> = entry_counts
397        .iter()
398        .map(|&entry_count| entry_count as usize)
399        .collect();
400
401    Ok((buffer, entry_counts))
402}
runefs/index.rs

runefs/
index.rs
