idx/
util.rs

1use std::{sync::{Arc, Mutex}, collections::HashMap, fs::File, io::{Read, BufReader}};
2use bzip2::bufread::BzDecoder;
3use databuffer::DataBuffer;
4use crate::{Cache, CacheIndex};
5
6type ParserFun<T> = fn(DataBuffer) -> T;
7
8pub trait DefParser {
9    fn parse_bytes(bytes: Vec<u8>) -> Self where Self: Sized {
10        DefParser::parse_buff(DataBuffer::with_vec(bytes))
11    }
12
13    fn parse_buff(buffer: DataBuffer) -> Self;
14}
15
16/**
17  The [`DefProvider`] is going to be what you'll primarily use to implement definition decoders and things along those lines.
18
19  You will use the [`DefProvider::with(cache, index)`] method to construct a definition provider, along with your definition type.
20
21  Let's say, for example, we had the below definition:
22
23  ```ignore
24  #[derive(Default)]
25  struct DummyDefinition {
26      dummy_int: u32,
27      dummy_str: String
28  }
29  ```
30
31  Which resides in index 1. You would implement your decoder:
32
33  ```ignore
34  impl DefParser for DummyDefinition {
35    fn parse_buff(buffer: DataBuffer) -> Self {
36        let mut def = DummyDefinition::default();
37
38        let opcode: u8;
39
40        loop {
41            opcode = buffer.read_u8();
42
43            match opcode {
44                0 -> break,
45                1 -> def.dummy_int = buffer.read_u32(),
46                2 -> def.dummy_str = buffer.read_ntstr()
47            }
48        }
49
50        return def;
51    }
52  }
53  ```
54
55  You would then create a definition provider like so: 
56
57  ```ignore
58  use idx::*;
59  use idx::util::*;
60  use std::sync::{Arc,Mutex};
61
62  let cache = Arc::from(Mutex::from(Cache::from_path("test_cache")));
63
64  let dummy_def_provider = DefProvider::<DummyDefinition>::with(cache, 1);
65
66  let definition = dummy_def_provider.get_def(&3, &1); //returns the parsed definition from file 1 of archive 3.
67
68  ```
69
70  It is additionally recommended to make some additional trait that can turn, for example, and item ID into the appropriate archive and file IDs
71
72  I would also recommend using [`ContainerIdProvider`] as the type to be passed for the ID, as it can accept both u32 and String. But this is up to you.
73
74  ```ignore
75  pub trait IdFetch {
76      type DefType;
77
78      fn for_id(id: u32) -> &Self::DefType;
79  }
80
81  impl IdFetch for DefProvider<DummyDefinition> {
82      type DefType = DummyDefinition;
83
84      fn for_id(id: u32) -> &DummyDefinition {
85          let archive = id >> 8;
86          let file = id & 0xff;
87
88          return self.get_def(&archive, &file, id);
89      }
90  }
91  ```
92 */
93pub struct DefProvider<T> {
94    pub file_provider: FileProvider,
95    pub index: u32,
96    pub parser: Option<ParserFun<T>>,
97    def_cache: HashMap<u32, T>
98}
99
100impl <T: DefParser> DefProvider<T> {
101    pub fn with(cache: &Arc<Mutex<Cache>>, index: u32) -> Self {
102        Self {
103            file_provider: FileProvider::from(cache),
104            index,
105            parser: Some(T::parse_buff),
106            def_cache: HashMap::new()
107        }
108    }
109
110    pub fn get_def(&mut self, archive: &dyn ContainerIdProvider, file: &dyn ContainerIdProvider, id: u32) -> &T {
111        if self.def_cache.contains_key(&id) {
112            return self.def_cache.get(&id).unwrap();
113        }
114
115        self.file_provider.index(self.index);
116        self.file_provider.archive(archive);
117
118        let data = self.file_provider.request(file);
119
120        let parse = self.parser.unwrap();
121
122        let def = parse(data);
123
124        self.def_cache.insert(id, def);
125
126        return self.def_cache.get(&id).unwrap();
127    }
128
129}
130
131/**
132  The FileProvider is the primary method of retrieving raw data from the cache. 
133
134  In order to function correctly, an index, archive and file ID must be supplied.
135
136  The index is type [`usize`], and the archive and file ID can either be a u32 reference (&[`u32`]) or a String reference (&[`String`]).
137  
138  ```no_run
139  use idx::util::FileProvider;
140  use idx::util::CacheBuilder;
141
142  let cache = CacheBuilder::new()
143                .with_path("test_cache")
144                .build();
145                
146  let mut data_provider = FileProvider::from(&cache);
147  
148  data_provider.index(19).archive(&6);
149  let data = data_provider.request(&17); //Returns the raw data for file 17 in archive 6 of index 19.
150
151  assert_ne!(0, data.len());
152  ```
153*/
154pub struct FileProvider {
155    cache: Arc<Mutex<Cache>>,
156    index: u32,
157    archive: u32,
158    data_file: Arc<Mutex<BufReader<File>>>,
159    keys: Vec<i64>,
160}
161
162impl FileProvider {
163    pub fn from(cache: &Arc<Mutex<Cache>>) -> Self {
164        let dfile = match cache.lock() {
165            Ok(n) => n.data_file.clone(),
166
167            Err(e) => {
168                panic!("Unable to lock cache: {}", e);
169            }
170        };
171
172        Self {
173            cache: cache.clone(),
174            index: 0,
175            archive: 0,
176            data_file: dfile,
177            keys: Vec::new()
178        }
179    }
180
181    pub fn index(&mut self, index: u32) -> &mut Self {
182        self.index = index;
183        self
184    }
185
186    pub fn archive(&mut self, archive: &dyn ContainerIdProvider) -> &mut Self {
187        if self.index == 0 {
188            self.archive = archive.get_id(None);
189            println!("WARNING: archive was set before the index was! IDX: {}, ARCHIVE: {}. This will break archive access via name hashes!", self.index, self.archive);
190        }
191
192        {
193            let mut _cache = self.cache.lock().unwrap();
194            let index = _cache.index(self.index as usize).unwrap();
195            self.archive = archive.get_id(Some(index));
196        }
197        self
198    }
199
200    pub fn with_keys(&mut self, keys: Vec<i64>) {
201        self.keys = keys
202    }
203
204    pub fn request(&mut self, file: &dyn ContainerIdProvider) -> DataBuffer {
205        let file_id = file.get_id(None);
206
207        let file_data = match self.cache.lock() {
208            Ok(mut n) => match n.index(self.index as usize) {
209                Some(s) => match s.container_info.containers.get(&self.archive) {
210                    Some(c) => match c.file_containers.get(&file_id) {
211                        Some(n) => DataBuffer::from_bytes(&n.data),
212                        None => DataBuffer::new()
213                    }
214                    None => {
215                        println!("Invalid archive supplied?");
216                        return DataBuffer::new();
217                    }
218                },
219                None => {
220                    panic!("Index has no containers?");
221                }
222            },
223            Err(_) => {
224                panic!("Unable to lock cache!");
225            }
226        };
227
228        if file_data.len() != 0 {
229            file_data
230        } else {
231            self.load_requested_container_files();
232
233            let data = match self.cache.lock() {
234                Ok(mut n) => match n.index(self.index as usize) {
235                    Some(s) => match s.container_info.containers.get(&self.archive) {
236                        Some(c) => match c.file_containers.get(&file_id) {
237                            Some(n) => DataBuffer::from_bytes(&n.data),
238                            None => DataBuffer::new()
239                        }
240                        None => {
241                            println!("Invalid archive supplied?");
242                            DataBuffer::new()
243                        }
244                    },
245                    None => {
246                        panic!("Index has no containers?");
247                    }
248                },
249                Err(_) => {
250                    panic!("Unable to lock cache!");
251                }
252            };
253
254            data
255        }
256    }
257
258    fn load_requested_container_files(&mut self) {
259        let container_data = self.get_requested_container_data();
260        let file_info = self.get_container_file_info();
261
262        let mut read_pos = container_data.len() - 1;
263        let num_loops = container_data[read_pos];
264
265        read_pos -= (num_loops as usize) * (file_info.len() * 4);
266
267        let mut buffer = DataBuffer::from_bytes(&container_data);
268        buffer.set_rpos(read_pos as usize);
269
270        let mut cache = match self.cache.lock() {
271            Ok(n) => n,
272            Err(_) => return
273        };
274
275        let index = match cache.index(self.index as usize) {
276            Some(n) => n,
277            None => return
278        };
279
280        let archive = match index.container_info.containers.get_mut(&self.archive) {
281            Some(n) => n,
282            None => return
283        };
284
285        if file_info.len() == 1 {
286            if let Some(file_container) = archive.file_containers.get_mut(&file_info[0]) {
287                file_container.data = container_data;
288            }
289        } else {
290            let mut file_sizes = Vec::<i32>::new();
291            for _ in 0..(num_loops as usize) {
292                let mut offset = 0_i32;
293                for file_index in 0..(file_info.len() as usize){
294                    offset += buffer.read_i32();
295                    if file_sizes.len() == file_index {
296                        file_sizes.push(offset);
297                    } else {
298                        file_sizes[file_index] += offset;
299                    }
300                }
301            }
302
303            buffer.set_rpos(read_pos);
304
305            let mut offset = 0;
306            for _ in 0..(num_loops as usize) {
307                let mut data_read = 0;
308                for file_index in &file_info {
309                    data_read += buffer.read_i32();
310
311                    match archive.file_containers.get_mut(file_index) {
312                        Some(n) => {
313                            n.data.append(&mut container_data[(offset as usize)..((offset + data_read) as usize)].to_vec())
314                        },
315                        None => {
316                            println!("Unknown file id: {}", file_index);
317                            continue;
318                        }
319                    }
320
321                    offset += data_read;
322                }
323            }
324        }
325    }
326
327    fn get_requested_container_data(&mut self) -> Vec<u8> {
328        let mut _cache = self.cache.lock().unwrap();
329
330        let index = match _cache.index(self.index as usize) {
331            Some(n) => n,
332            None => {
333                return Vec::new();
334            }
335        };
336
337        let _ = match index.container_data(self.data_file.lock().unwrap(), self.archive) {
338            Some(n) => match decompress_container_data(n) {
339                Some(n) => return n,
340                None => return Vec::new()
341            },
342            None => return Vec::new()
343        };
344    }
345
346    fn get_container_file_info(&mut self) -> Vec<u32> {
347        let mut file_info = Vec::<u32>::new();
348
349        let mut _cache = self.cache.lock().unwrap();
350
351        let index = match _cache.index(self.index as usize) {
352            Some(n) => n,
353            None => {
354                return Vec::new();
355            }
356        };
357
358        let container = match index.container_info.containers.get(&self.archive) {
359            Some(n) => n,
360            None => return Vec::new()
361        };
362
363        for file in container.file_indices.iter() {
364            file_info.push(*file);
365        }
366
367        file_info
368    }
369}
370
371pub trait ContainerIdProvider {
372    fn get_id(&self, _: Option<&mut CacheIndex>) -> u32;
373}
374
375impl ContainerIdProvider for String {
376    fn get_id(&self, idx: Option<&mut CacheIndex>) -> u32 {
377        let hash = get_name_hash(&self);
378
379        if let Some(index) = idx {
380            index.get_container_by_name_hash(hash)
381        } else {
382            hash
383        }
384    }
385}
386
387impl ContainerIdProvider for u32 {
388    fn get_id(&self, _: Option<&mut CacheIndex>) -> u32 {
389        *self
390    }
391}
392
393fn get_name_hash(name: &str) -> u32 {
394    let name_clean = name.to_lowercase();
395
396    let mut hash = 0;
397
398    for char in name_clean.into_bytes() {
399        hash = (char as u32) + ((hash << 5) - hash);
400    }
401
402    hash
403}
404
405pub(crate) fn decompress_container_data(packed_data: Vec<u8>) -> Option<Vec<u8>> {
406    let mut data = DataBuffer::with_vec(packed_data);
407    let mut unpacked = Vec::<u8>::new();
408
409    if data.len() == 0 {
410        return Some(Vec::new());
411    }
412
413    let compression = data.read_u8();
414    let container_size = data.read_u32();
415
416    if container_size > 5000000 {
417        println!("Invalid container size! {}", container_size);
418        None
419    } else {
420        match compression {
421            0 => { //Uncompressed
422                let trim_at = data.get_rpos();
423                let mut raw = data.deconstruct();
424
425                raw.drain(..trim_at);
426                Some(raw)
427            },
428
429            1 => { //Bzip2 (supposedly)
430                let decompressed_size = data.read_u32();
431                let trim_at = data.get_rpos() - 4;
432
433                let mut trimmed_data = data.deconstruct();
434                trimmed_data.drain(..trim_at);
435
436                //Re-add header jagex strips.
437                trimmed_data[0] = b'B';
438                trimmed_data[1] = b'Z';
439                trimmed_data[2] = b'h';
440                trimmed_data[3] = b'1';
441
442                match BzDecoder::new(&trimmed_data[..]).read_to_end(&mut unpacked) {
443                    Ok(_) => {},
444                    Err(e) => {
445                        println!("Bzip2 Decompression Error: {}", e);
446                    }
447                }
448
449                assert_eq!(decompressed_size, unpacked.len() as u32);
450                Some(unpacked)
451            },
452
453            _ => { //DEFLATE/Gzip/Zip
454                let decompressed_size = data.read_u32();
455                data.set_rpos(data.get_rpos() + 10);
456                let trim_at = data.get_rpos();
457
458                let mut trimmed_data = data.deconstruct();
459                trimmed_data.drain(..trim_at);
460
461                unpacked = match inflate::inflate_bytes(&trimmed_data) {
462                    Ok(n) => n,
463                    Err(e) => {
464                        println!("Error deflating gzip-compressed cache data: {}", e);
465                        return None;
466                    }
467                };
468
469                assert_eq!(decompressed_size, unpacked.len() as u32);
470                Some(unpacked)
471            }
472        }
473    }
474}
475
476pub struct CacheBuilder {
477    pub cache_path: String,
478    pub base_file_name: String,
479    pub calculate_crc32: bool
480}
481
482impl Default for CacheBuilder {
483    fn default() -> Self {
484        Self {
485            cache_path: String::new(),
486            base_file_name: String::from("main_file_cache"),
487            calculate_crc32: true
488        }
489    }
490}
491
492impl CacheBuilder {
493    pub fn new() -> Self {
494        Self::default()
495    }
496
497    /// Sets the path to the cache folder. Note: this must be a path to a **folder**, not a file.
498    pub fn with_path(mut self, path: &str) -> Self {
499        self.cache_path = String::from(path);
500        self
501    }
502
503    /// Sets the base name for cache files. Default is "main_file_cache"
504    pub fn with_base_filename(mut self, filename: &str) -> Self {
505        self.base_file_name = String::from(filename);
506        self
507    }
508
509    /// Decides whether or not to calculate crc sums for archives. Defaults to true.
510    pub fn calculate_crc32(mut self, calculate: bool) -> Self {
511        self.calculate_crc32 = calculate;
512        self
513    }
514
515    pub fn build(self) -> std::sync::Arc<std::sync::Mutex<Cache>> {
516        let cache = Cache::with(self).unwrap();
517        Arc::from(Mutex::from(cache))
518    } 
519}
idx/util.rs

idx/
util.rs
