genie_dat/
lib.rs

1//! A reader for Age of Empires game data files.
2//!
3//! This crate aims to support every data file that exists, but is for now being tested with AoE1,
4//! AoE2, and AoE2: HD Edition.
5
6#![deny(future_incompatible)]
7#![deny(nonstandard_style)]
8#![deny(rust_2018_idioms)]
9#![deny(unsafe_code)]
10#![warn(missing_docs)]
11#![warn(unused)]
12
13mod civ;
14mod color_table;
15mod random_map;
16mod sound;
17mod sprite;
18mod task;
19mod tech;
20mod tech_tree;
21mod terrain;
22mod unit_type;
23
24use byteorder::{ReadBytesExt, WriteBytesExt, LE};
25pub use civ::{Civilization, CivilizationID};
26pub use color_table::{ColorTable, PaletteIndex};
27use flate2::{read::DeflateDecoder, write::DeflateEncoder, Compression};
28use genie_support::{f32_eq, ReadSkipExt, TechID};
29pub use random_map::*;
30pub use sound::{Sound, SoundID, SoundItem};
31pub use sprite::{GraphicID, SoundProp, Sprite, SpriteAttackSound, SpriteDelta, SpriteID};
32use std::cmp::{Ordering, PartialOrd};
33use std::convert::TryInto;
34use std::fmt;
35use std::io::{Read, Result, Write};
36pub use task::{Task, TaskList};
37pub use tech::{Tech, TechEffect};
38pub use tech_tree::{
39    ParseTechTreeTypeError, TechTree, TechTreeAge, TechTreeBuilding, TechTreeDependencies,
40    TechTreeStatus, TechTreeTech, TechTreeType, TechTreeUnit,
41};
42pub use terrain::{
43    Terrain, TerrainAnimation, TerrainBorder, TerrainID, TerrainObject, TerrainPassGraphic,
44    TerrainRestriction, TerrainSpriteFrame, TileSize,
45};
46pub use unit_type::*;
47
48/// A game version targeted by a data file.
49#[derive(Debug, Clone, Copy, PartialEq, Eq)]
50pub enum GameVersion {
51    /// The original expansion-less Age of Empires 2: Age of Kings.
52    AoK,
53    /// Age of Empires 2 with the The Conquerors expansion.
54    AoC,
55    /// Age of EMpires 2: HD Edition.
56    HD,
57}
58
59impl GameVersion {
60    /// Get the most likely internal game data version number for a given game version.
61    fn as_f32(self) -> f32 {
62        use GameVersion::*;
63        match self {
64            AoK => 11.5,
65            AoC => 11.97,
66            HD => 12.0,
67        }
68    }
69}
70
71/// A data file version.
72#[derive(Debug, Clone, Copy, PartialEq)]
73pub struct FileVersion([u8; 8]);
74
75impl From<[u8; 8]> for FileVersion {
76    fn from(identifier: [u8; 8]) -> Self {
77        assert!(matches!(
78            identifier,
79            // "VER *.*\0"
80            [b'V', b'E', b'R', b' ', b'0'..=b'9', b'.', b'0'..=b'9', 0]
81        ));
82        Self(identifier)
83    }
84}
85
86impl From<&str> for FileVersion {
87    fn from(string: &str) -> Self {
88        assert!(string.len() <= 8);
89        let mut bytes = [0; 8];
90        (&mut bytes[..string.len()]).copy_from_slice(string.as_bytes());
91        Self::from(bytes)
92    }
93}
94
95impl fmt::Display for FileVersion {
96    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
97        match std::str::from_utf8(&self.0[0..7]) {
98            Ok(s) => write!(f, "{}", s),
99            Err(_) => write!(f, "{:?}", self.0),
100        }
101    }
102}
103
104impl PartialOrd for FileVersion {
105    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
106        match self.major_version().partial_cmp(&other.major_version()) {
107            None | Some(Ordering::Equal) => {
108                self.minor_version().partial_cmp(&other.minor_version())
109            }
110            Some(order) => Some(order),
111        }
112    }
113}
114
115impl FileVersion {
116    /// Get the major version component, eg the 5 in "VER 5.8".
117    fn major_version(self) -> u8 {
118        self.0[4] - b'0'
119    }
120    /// Get the minor version component, eg the 8 in "VER 5.8".
121    fn minor_version(self) -> u8 {
122        self.0[6] - b'0'
123    }
124
125    /// Is this file built for Star Wars: Galactic Battlegrounds?
126    pub fn is_swgb(self) -> bool {
127        false
128    }
129
130    /// Is this file built for Age of Empires II: The Conquerors?
131    pub fn is_aoc(self) -> bool {
132        let data_version = self.into_data_version();
133        f32_eq!(data_version, 11.97)
134    }
135
136    /// Is this file built for Age of Empires II: Definitive Edition?
137    pub fn is_de2(self) -> bool {
138        self >= FileVersion(*b"VER 5.8\0")
139    }
140
141    /// Get the data version associated with this file version.
142    fn into_data_version(self) -> f32 {
143        match &self.0 {
144            b"VER 5.7\0" => 11.97,
145            _ => panic!("unknown version"),
146        }
147    }
148}
149
150/// A data file.
151#[derive(Debug, Clone)]
152pub struct DatFile {
153    file_version: FileVersion,
154    game_version: GameVersion,
155    /// Terrain restriction tables.
156    pub terrain_tables: Vec<TerrainRestriction>,
157    /// Tile size data.
158    pub tile_sizes: Vec<TileSize>,
159    /// Terrains.
160    pub terrains: Vec<Terrain>,
161    /// Terrain border data, specifying how different terrains blend.
162    pub terrain_borders: Vec<TerrainBorder>,
163    /// Random map data from AoE1.
164    random_maps: Vec<RandomMapInfo>,
165    /// Data about player colours.
166    pub color_tables: Vec<ColorTable>,
167    /// The available sounds.
168    pub sounds: Vec<Sound>,
169    /// The available sprites.
170    pub sprites: Vec<Option<Sprite>>,
171    /// Tech effect data.
172    pub effects: Vec<TechEffect>,
173    /// Task lists for unit types.
174    pub task_lists: Vec<Option<TaskList>>,
175    /// The available civilizations.
176    pub civilizations: Vec<Civilization>,
177    /// Techs or researches.
178    pub techs: Vec<Tech>,
179    /// Tech tree data.
180    pub tech_tree: TechTree,
181}
182
183impl DatFile {
184    /// Read a data file from a compressed byte stream.
185    pub fn read_from(input: impl Read) -> Result<Self> {
186        let mut input = DeflateDecoder::new(input);
187
188        let mut file_version = [0u8; 8];
189        input.read_exact(&mut file_version)?;
190        let file_version = FileVersion(file_version);
191
192        let num_terrain_tables = input.read_u16::<LE>()?;
193        let num_terrains = input.read_u16::<LE>()?;
194
195        let game_version = if file_version == FileVersion(*b"VER 5.7\0") {
196            match num_terrains {
197                32 => GameVersion::AoK,
198                41 => GameVersion::AoC,
199                100 => GameVersion::HD,
200                _ => GameVersion::AoC, // TODO support different versions
201            }
202        } else {
203            GameVersion::AoC // TODO support different versions
204        };
205
206        // AoC hardcodes to 42 terrains, but says 41 terrains in the data file.
207        // The 42nd terrain is zeroed out.
208        let num_terrains_fixed = if game_version == GameVersion::AoC && num_terrains == 41 {
209            42
210        } else {
211            num_terrains
212        };
213
214        // Two lists of pointers
215        input.skip(4 * u64::from(num_terrain_tables) + 4 * u64::from(num_terrain_tables))?;
216
217        #[must_use]
218        fn read_array<T>(num: usize, mut read: impl FnMut() -> Result<T>) -> Result<Vec<T>> {
219            let mut list = vec![];
220            for _ in 0..num {
221                list.push(read()?);
222            }
223            Ok(list)
224        }
225
226        let terrain_tables = read_array(num_terrain_tables.into(), || {
227            TerrainRestriction::read_from(&mut input, file_version, num_terrains)
228        })?;
229
230        let num_color_tables = input.read_u16::<LE>()?;
231        let color_tables = read_array(num_color_tables.into(), || {
232            ColorTable::read_from(&mut input)
233        })?;
234
235        let num_sounds = input.read_u16::<LE>()?;
236        let sounds = read_array(num_sounds.into(), || {
237            Sound::read_from(&mut input, file_version)
238        })?;
239
240        let num_sprites = input.read_u16::<LE>()?;
241        let sprites_exist = read_array(num_sprites.into(), || {
242            input.read_u32::<LE>().map(|n| n != 0)
243        })?;
244        let mut sprites = vec![];
245        for exists in sprites_exist {
246            sprites.push(if exists {
247                Some(Sprite::read_from(&mut input)?)
248            } else {
249                None
250            });
251        }
252
253        // Some raw pointer values
254        let _map_vtable_pointer = input.read_i32::<LE>()?;
255        let _tiles_pointer = input.read_i32::<LE>()?;
256
257        // Bogus stuff
258        let _map_width = input.read_i32::<LE>()?;
259        let _map_height = input.read_i32::<LE>()?;
260        let _world_width = input.read_i32::<LE>()?;
261        let _world_height = input.read_i32::<LE>()?;
262
263        let mut tile_sizes = vec![TileSize::default(); 19];
264        for val in tile_sizes.iter_mut() {
265            *val = TileSize::read_from(&mut input)?;
266        }
267
268        // Padding
269        input.read_i16::<LE>()?;
270
271        let terrains = read_array(num_terrains_fixed.into(), || {
272            Terrain::read_from(&mut input, file_version, num_terrains_fixed)
273        })?;
274        let terrain_borders = read_array(16, || TerrainBorder::read_from(&mut input))?;
275
276        // Should just skip all this shit probably
277        let _map_row_offset = input.read_i32::<LE>()?;
278        let _map_min_x = input.read_f32::<LE>()?;
279        let _map_min_y = input.read_f32::<LE>()?;
280        let _map_max_x = input.read_f32::<LE>()?;
281        let _map_max_y = input.read_f32::<LE>()?;
282        let _map_max_x = input.read_f32::<LE>()?;
283        let _map_max_y = input.read_f32::<LE>()?;
284        let _additional_terrain_count = input.read_u16::<LE>()?;
285        let _borders_used = input.read_u16::<LE>()?;
286        let _max_terrain = input.read_u16::<LE>()?;
287        let _tile_width = input.read_u16::<LE>()?;
288        let _tile_height = input.read_u16::<LE>()?;
289        let _tile_half_width = input.read_u16::<LE>()?;
290        let _tile_half_height = input.read_u16::<LE>()?;
291        let _elev_height = input.read_u16::<LE>()?;
292        let _current_row = input.read_u16::<LE>()?;
293        let _current_column = input.read_u16::<LE>()?;
294        let _block_begin_row = input.read_u16::<LE>()?;
295        let _block_end_row = input.read_u16::<LE>()?;
296        let _block_begin_column = input.read_u16::<LE>()?;
297        let _block_end_column = input.read_u16::<LE>()?;
298        let _seach_map_pointer = input.read_i32::<LE>()?;
299        let _seach_map_rows_pointer = input.read_i32::<LE>()?;
300        let _any_frame_change = input.read_u8()?;
301        let _map_visible = input.read_u8()?;
302        let _map_fog_of_war = input.read_u8()?;
303
304        // Lots more pointers and stuff
305        input.skip(21 + 157 * 4)?;
306
307        let num_random_maps = input.read_u32::<LE>()? as usize;
308        let _random_maps_pointer = input.read_u32::<LE>()?;
309
310        let mut random_maps = read_array(num_random_maps, || RandomMapInfo::read_from(&mut input))?;
311        for map in random_maps.iter_mut() {
312            map.finish(&mut input)?;
313        }
314
315        let num_effects = input.read_u32::<LE>()? as usize;
316        let effects = read_array(num_effects, || TechEffect::read_from(&mut input))?;
317
318        let num_task_lists = input.read_u32::<LE>()? as usize;
319        let task_lists = read_array(num_task_lists, || {
320            if input.read_u8()? != 0 {
321                TaskList::read_from(&mut input).map(Some)
322            } else {
323                Ok(None)
324            }
325        })?;
326
327        let num_civilizations = input.read_u16::<LE>()?;
328        let civilizations = read_array(num_civilizations.into(), || {
329            let player_type = input.read_i8()?;
330            assert_eq!(player_type, 1);
331            Civilization::read_from(&mut input, game_version)
332        })?;
333
334        let num_techs = input.read_u16::<LE>()?;
335        let techs = read_array(num_techs.into(), || Tech::read_from(&mut input))?;
336
337        let _time_slice = input.read_u32::<LE>()?;
338        let _unit_kill_rate = input.read_u32::<LE>()?;
339        let _unit_kill_total = input.read_u32::<LE>()?;
340        let _unit_hit_point_rate = input.read_u32::<LE>()?;
341        let _unit_hit_point_total = input.read_u32::<LE>()?;
342        let _razing_kill_rate = input.read_u32::<LE>()?;
343        let _razing_kill_total = input.read_u32::<LE>()?;
344
345        let tech_tree = TechTree::read_from(&mut input)?;
346
347        Ok(Self {
348            file_version,
349            game_version,
350            terrain_tables,
351            tile_sizes,
352            terrains,
353            terrain_borders,
354            random_maps,
355            color_tables,
356            sounds,
357            sprites,
358            effects,
359            task_lists,
360            civilizations,
361            techs,
362            tech_tree,
363        })
364    }
365
366    /// Serialize this data file to an output stream. Compression is applied by this function.
367    pub fn write_to<W: Write>(&self, output: &mut W) -> Result<()> {
368        let num_terrains = if self.game_version == GameVersion::AoC && self.terrains.len() == 42 {
369            41
370        } else {
371            self.terrains.len()
372        };
373
374        let mut output = DeflateEncoder::new(output, Compression::default());
375        output.write_all(&self.file_version.0)?;
376        output.write_u16::<LE>(self.terrain_tables.len().try_into().unwrap())?;
377        output.write_u16::<LE>(num_terrains.try_into().unwrap())?;
378
379        // Two lists of pointers
380        output.write_all(&vec![
381            0u8;
382            4 * self.terrain_tables.len()
383                + 4 * self.terrain_tables.len()
384        ])?;
385
386        for table in &self.terrain_tables {
387            table.write_to(
388                &mut output,
389                self.file_version,
390                num_terrains.try_into().unwrap(),
391            )?;
392        }
393
394        output.write_u16::<LE>(self.color_tables.len().try_into().unwrap())?;
395        for table in &self.color_tables {
396            table.write_to(&mut output)?;
397        }
398
399        output.write_u16::<LE>(self.sounds.len().try_into().unwrap())?;
400        for sound in &self.sounds {
401            sound.write_to(&mut output, self.file_version)?;
402        }
403
404        output.write_u16::<LE>(self.sprites.len().try_into().unwrap())?;
405        for maybe_sprite in &self.sprites {
406            output.write_u32::<LE>(match maybe_sprite {
407                Some(_) => 1,
408                None => 0,
409            })?;
410        }
411        for maybe_sprite in &self.sprites {
412            if let Some(sprite) = maybe_sprite {
413                sprite.write_to(&mut output)?;
414            }
415        }
416
417        output.write_u32::<LE>(0)?; // map vtable pointer
418        output.write_u32::<LE>(0)?; // map tiles pointer
419        output.write_u32::<LE>(0)?; // map width
420        output.write_u32::<LE>(0)?; // map height
421        output.write_u32::<LE>(0)?; // world width
422        output.write_u32::<LE>(0)?; // world height
423
424        for size in &self.tile_sizes {
425            size.write_to(&mut output)?;
426        }
427
428        // Padding
429        output.write_i16::<LE>(0)?;
430
431        for terrain in &self.terrains {
432            terrain.write_to(&mut output, self.file_version, self.terrains.len() as u16)?;
433        }
434        for border in &self.terrain_borders {
435            border.write_to(&mut output)?;
436        }
437
438        // TODO put correct values in
439        output.write_i32::<LE>(0)?;
440        output.write_f32::<LE>(0.0)?;
441        output.write_f32::<LE>(0.0)?;
442        output.write_f32::<LE>(0.0)?;
443        output.write_f32::<LE>(0.0)?;
444        output.write_f32::<LE>(0.0)?;
445        output.write_f32::<LE>(0.0)?;
446        output.write_u16::<LE>(0)?;
447        output.write_u16::<LE>(0)?;
448        output.write_u16::<LE>(0)?;
449        output.write_u16::<LE>(0)?;
450        output.write_u16::<LE>(0)?;
451        output.write_u16::<LE>(0)?;
452        output.write_u16::<LE>(0)?;
453        output.write_u16::<LE>(0)?;
454        output.write_u16::<LE>(0)?;
455        output.write_u16::<LE>(0)?;
456        output.write_u16::<LE>(0)?;
457        output.write_u16::<LE>(0)?;
458        output.write_u16::<LE>(0)?;
459        output.write_u16::<LE>(0)?;
460        output.write_i32::<LE>(0)?;
461        output.write_i32::<LE>(0)?;
462        output.write_u8(0)?;
463        output.write_u8(0)?;
464        output.write_u8(0)?;
465
466        // Lots more pointers and stuff
467        let nulls = [0; 21 + 157 * 4];
468        output.write_all(&nulls)?;
469
470        output.write_u32::<LE>(self.random_maps.len() as u32)?;
471        output.write_u32::<LE>(0)?; // pointer
472
473        for map in &self.random_maps {
474            map.write_to(&mut output)?;
475        }
476        for map in &self.random_maps {
477            map.write_commands_to(&mut output)?;
478        }
479
480        output.write_u32::<LE>(self.effects.len() as u32)?;
481        for effect in &self.effects {
482            effect.write_to(&mut output)?;
483        }
484
485        output.write_u32::<LE>(self.task_lists.len() as u32)?;
486        for task_list in &self.task_lists {
487            if let Some(task_list) = task_list {
488                output.write_u8(1)?;
489                task_list.write_to(&mut output)?;
490            } else {
491                output.write_u8(0)?;
492            }
493        }
494
495        output.write_u16::<LE>(self.civilizations.len() as u16)?;
496        for civilization in &self.civilizations {
497            output.write_i8(1)?; // player type
498            civilization.write_to(&mut output, self.game_version)?;
499        }
500
501        output.write_u16::<LE>(self.techs.len() as u16)?;
502        for tech in &self.techs {
503            tech.write_to(&mut output)?;
504        }
505
506        output.write_u32::<LE>(0)?;
507        output.write_u32::<LE>(0)?;
508        output.write_u32::<LE>(0)?;
509        output.write_u32::<LE>(0)?;
510        output.write_u32::<LE>(0)?;
511        output.write_u32::<LE>(0)?;
512        output.write_u32::<LE>(0)?;
513
514        self.tech_tree.write_to(&mut output)?;
515
516        Ok(())
517    }
518
519    /// Get a tech by its ID.
520    pub fn get_tech(&self, id: impl Into<TechID>) -> Option<&Tech> {
521        let id: TechID = id.into();
522        self.techs.get(usize::from(id))
523    }
524
525    /// Get a terrain type by its ID.
526    pub fn get_terrain(&self, id: impl Into<TerrainID>) -> Option<&Terrain> {
527        let id: TerrainID = id.into();
528        self.terrains.get(usize::from(id))
529    }
530
531    /// Get the GAIA civilization.
532    pub fn get_gaia(&self) -> Option<&Civilization> {
533        self.get_civilization(0)
534    }
535
536    /// Get a civilization by its ID.
537    pub fn get_civilization(&self, id: impl Into<CivilizationID>) -> Option<&Civilization> {
538        let id: CivilizationID = id.into();
539        self.civilizations.get(usize::from(id))
540    }
541
542    /// Get a sound by its ID.
543    pub fn get_sound(&self, id: impl Into<SoundID>) -> Option<&Sound> {
544        let id: SoundID = id.into();
545        self.sounds.get(usize::from(id))
546    }
547
548    /// Get a sprite by its ID.
549    pub fn get_sprite(&self, id: impl Into<SpriteID>) -> Option<&Sprite> {
550        let id: SpriteID = id.into();
551        self.sprites.get(usize::from(id)).and_then(Option::as_ref)
552    }
553}
554
555#[cfg(test)]
556mod tests {
557    use super::*;
558    use std::{
559        collections::hash_map::DefaultHasher,
560        fs::File,
561        hash::{Hash, Hasher},
562        io::Cursor,
563    };
564
565    #[test]
566    fn aok() -> anyhow::Result<()> {
567        let mut f = File::open("fixtures/aok.dat")?;
568        let dat = DatFile::read_from(&mut f)?;
569        assert_eq!(dat.civilizations.len(), 14);
570        Ok(())
571    }
572
573    #[test]
574    fn aoc() -> anyhow::Result<()> {
575        let mut f = File::open("fixtures/aoc1.0c.dat")?;
576        let dat = DatFile::read_from(&mut f)?;
577        assert_eq!(dat.civilizations.len(), 19);
578        Ok(())
579    }
580
581    #[test]
582    fn non_7bit_ascii_tech_name() -> anyhow::Result<()> {
583        let mut f = File::open("fixtures/age-of-chivalry.dat")?;
584        let dat = DatFile::read_from(&mut f)?;
585        assert_eq!(dat.techs[859].name(), "Székely (enable)");
586        Ok(())
587    }
588
589    #[test]
590    fn hd_edition() -> anyhow::Result<()> {
591        let mut f = File::open("fixtures/hd.dat")?;
592        let dat = DatFile::read_from(&mut f)?;
593        assert_eq!(dat.civilizations.len(), 32);
594        Ok(())
595    }
596
597    #[test]
598    fn reserialize() -> anyhow::Result<()> {
599        let original = std::fs::read("fixtures/aoc1.0c.dat")?;
600        let mut cursor = Cursor::new(&original);
601        let dat = DatFile::read_from(&mut cursor)?;
602        let mut serialized = vec![];
603        dat.write_to(&mut serialized)?;
604
605        let mut cursor = Cursor::new(&serialized);
606        let dat2 = DatFile::read_from(&mut cursor)?;
607
608        let mut orig_hasher = DefaultHasher::new();
609        let mut new_hasher = DefaultHasher::new();
610        format!("{:?}", dat).hash(&mut orig_hasher);
611        format!("{:?}", dat2).hash(&mut new_hasher);
612        assert_eq!(orig_hasher.finish(), new_hasher.finish());
613
614        Ok(())
615    }
616}
genie_dat/lib.rs

genie_dat/
lib.rs
