wot_datfile_parser/
parser.rs

1use std::collections::BTreeMap;
2pub use std::{
3    collections::{HashMap, HashSet},
4    rc::Rc,
5};
6
7use itertools::Itertools;
8use serde_json::Value as JSONValue;
9use serde_pickle::{HashableValue, Value as PickleValue};
10
11use crate::{fields::gen_collection, manual_parser::pickle_val_to_json_manual, InterceptFn, Result};
12type Dict<T> = HashMap<String, T>;
13use crate::{
14    battle_results::{Field, FieldType},
15    error::Error,
16    fields::{matches_version, FieldCollection},
17    to_default_if_none, Battle,
18};
19
20
21pub struct DatFileParser {
22    /// Identifier manager. Identifier lists can be retrieved with a checksum
23    /// value
24    fields: FieldCollection,
25}
26
27/// The raw data structure from the datfile is not very easy to work with. So we
28/// break it down into the following structure
29struct DatfileFormat {
30    arena_unique_id: String,
31    account_self:    Vec<PickleValue>,
32    vehicle_self:    HashMap<String, Vec<PickleValue>>,
33
34    common:      Vec<PickleValue>,
35    account_all: HashMap<String, Vec<PickleValue>>,
36    vehicle_all: HashMap<String, Vec<PickleValue>>,
37    player_info: HashMap<String, Vec<PickleValue>>,
38}
39
40
41#[derive(Debug)]
42pub enum Intercept {
43    Success(&'static Field, serde_json::Value),
44    NotPresent(&'static Field, serde_json::Value),
45    ManuallyParsed(&'static Field, serde_json::Value),
46    Failed(&'static Field, serde_json::Value, String),
47}
48
49impl Intercept {
50    pub fn original_result(self) -> serde_json::Value {
51        use Intercept::*;
52        match self {
53            Success(_, val) | NotPresent(_, val) | ManuallyParsed(_, val) | Failed(_, val, _) => val,
54        }
55    }
56}
57
58impl DatFileParser {
59    /// Parse a datfile into a Battle struct
60    pub fn parse(&self, input: &[u8]) -> Result<Battle> {
61        // Load the root pickle
62        let root_pickle = utils::load_pickle(input).unwrap();
63
64        // Convert the deeply nested root pickle into objects that can be easily parsed
65        let datfile_format = parse_root_pickle(root_pickle).unwrap();
66
67        // Parse the pickle objects to make a battle
68        self.parse_datfile_format(datfile_format, |intercept, _| intercept.original_result())
69    }
70
71    /// Same as `parse` but takes a function that you can use to implement your own parsing code to convert a
72    /// pickle value to its JSON counterpart
73    pub fn parse_intercept(&self, input: &[u8], intercept: InterceptFn) -> Result<Battle> {
74        // Load the root pickle
75        let root_pickle = utils::load_pickle(input).unwrap();
76
77        // Convert the deeply nested root pickle into objects that can be easily parsed
78        let datfile_format = parse_root_pickle(root_pickle).unwrap();
79
80        // Parse the pickle objects to make a battle
81        self.parse_datfile_format(datfile_format, intercept)
82    }
83
84    /// Construct a parser. You can then use this parser to parse any number of datfiles
85    pub fn new() -> Self {
86        Self {
87            fields: gen_collection(),
88        }
89    }
90
91    fn parse_datfile_format(&self, datfile: DatfileFormat, intercept: InterceptFn) -> Result<Battle> {
92        use FieldType::*;
93
94        let arena_unique_id = datfile.arena_unique_id;
95
96        let common = pickle_to_json(&self.fields, Common, datfile.common, intercept)?;
97
98        let account_self = pickle_to_json(&self.fields, AccountSelf, datfile.account_self, intercept)?;
99        let account_self = HashMap::from([(
100            account_self.pointer("/accountDBID").unwrap().to_string(),
101            account_self,
102        )]);
103
104        let vehicle_self = parse_list(&self.fields, VehicleSelf, datfile.vehicle_self, intercept)?;
105        let player_info = parse_list(&self.fields, PlayerInfo, datfile.player_info, intercept)?;
106        let account_all = parse_list(&self.fields, AccountAll, datfile.account_all, intercept)?;
107        let vehicle_all = parse_list(&self.fields, VehicleAll, datfile.vehicle_all, intercept)?;
108
109        Ok(Battle {
110            arena_unique_id,
111            common,
112            player_info,
113            account_all,
114            vehicle_all,
115            vehicle_self,
116            account_self,
117        })
118    }
119}
120
121fn parse_list(
122    fields: &FieldCollection, field_type: FieldType, input: Dict<Vec<PickleValue>>, intercept: InterceptFn,
123) -> Result<Dict<JSONValue>> {
124    input
125        .into_iter()
126        .map(|(key, value)| {
127            pickle_to_json(fields, field_type.clone(), value, intercept).map(|value| (key, value))
128        })
129        .collect()
130}
131
132fn decompress_and_load_pickle(input: &PickleValue) -> Result<PickleValue> {
133    let PickleValue::Bytes(input) = input else { return Err(Error::PickleFormatError) };
134    let decompressed =
135        miniz_oxide::inflate::decompress_to_vec_zlib(input).map_err(|_| Error::DecompressionError)?;
136
137    Ok(serde_pickle::value_from_slice(&decompressed, Default::default())?)
138}
139
140fn parse_root_pickle(root_pickle: PickleValue) -> Result<DatfileFormat> {
141    use PickleValue::*;
142    // root pickle is a tuple of the shape : (i64, Tuple)
143    let Tuple(root_tuple) = root_pickle else { return Err(Error::PickleFormatError) };
144
145    // data tuple should contain the following: (arenaUniqueID, [u8], [u8], [u8])
146    // the three u8 buffers in this tuple are compressed pickle dumps
147    let [_, Tuple(data_tuple)] = root_tuple.as_slice() else { return Err(Error::PickleFormatError) };
148
149    let [I64(arena_unique_id), rest @ ..] = data_tuple.as_slice() else {
150        return Err(Error::PickleFormatError)
151    };
152
153    let Some((List(account_self), Dict(vehicle_self), Tuple(multiple))) = rest.into_iter().map(decompress_and_load_pickle).flatten().next_tuple() else {
154        return Err(Error::PickleFormatError)
155    };
156
157    let Some((List(common), Dict(player_info), Dict(vehicle_all), Dict(account_all))) = multiple.into_iter().next_tuple() else {
158        return Err(Error::PickleFormatError)
159    };
160
161    Ok(DatfileFormat {
162        arena_unique_id: arena_unique_id.to_string(),
163        account_self,
164        common,
165        account_all: to_rust_dict(account_all)?,
166        vehicle_all: to_rust_dict(vehicle_all)?,
167        player_info: to_rust_dict(player_info)?,
168        vehicle_self: to_rust_dict(vehicle_self)?,
169    })
170}
171
172
173fn pickle_to_json(
174    fields: &FieldCollection, field_type: FieldType, value_list: Vec<PickleValue>, intercept: InterceptFn,
175) -> Result<JSONValue> {
176    let mut value_list = value_list.into_iter();
177
178    // The checksum describes the list of identifiers that are associated with that list of PickleValue.
179    // This prevents us from blindly assigning, for example `damageDealt` identifier to
180    // `PickleValue::I64(5433)` because `5433` looks like a `damageDealt` value. With checksum we
181    // can know for sure.
182    let Some(PickleValue::I64(checksum)) = value_list.next() else {
183        return Err(Error::OtherError("Value list is empty"))
184    };
185
186    // If we cannot find the correct the identifier list, we cannot parse the
187    // datfile so we return with error
188    let (iden_list, version) = fields
189        .get_fields_list(checksum)
190        .ok_or_else(|| Error::UnknownChecksum(field_type.to_str(), checksum))?;
191
192    let mut map = HashMap::new();
193    for iden in iden_list {
194        if !matches_version(version, iden) {
195            let value = intercept(
196                Intercept::NotPresent(iden, iden.default.to_json_value()),
197                PickleValue::None,
198            );
199
200            map.insert(iden.name, value);
201        } else {
202            let value = value_list.next().ok_or_else(|| Error::DecompressionError)?;
203
204            map.insert(iden.name, pickle_val_to_json(iden, value, intercept));
205        }
206    }
207
208    assert!(value_list.next().is_none());
209    Ok(serde_json::to_value(map).unwrap())
210}
211
212
213/// Convert a `PickleValue` that contains a field value(for ex. field value
214/// of `damageDealt` is of type `i32`) to JSON. Note that even if the
215/// parsing fails we get a JSON because it will be the default value for
216/// the field We make the distinction between `Ok` and `Err` based on
217/// whether the field value was parsed succesfully to JSON
218fn pickle_val_to_json(iden: &'static Field, input: PickleValue, intercept: InterceptFn) -> JSONValue {
219    let value = to_default_if_none(iden, input);
220
221    match serde_pickle::from_value(value.clone()) {
222        Ok(json_value) => intercept(Intercept::Success(iden, json_value), value),
223
224        // Simple parsing did not work so we delegate to the more
225        // powerful manual parser
226        Err(_) => match pickle_val_to_json_manual(value.clone(), iden) {
227            Ok(json_value) => intercept(Intercept::ManuallyParsed(iden, json_value), value),
228            Err((err, json_value)) => intercept(Intercept::Failed(iden, json_value, err.to_string()), value),
229        },
230    }
231}
232
233
234fn to_rust_dict(input: BTreeMap<HashableValue, PickleValue>) -> Result<Dict<Vec<PickleValue>>> {
235    input
236        .into_iter()
237        .map(|(key, value)| match value {
238            PickleValue::List(list) => Ok((key.to_string(), list)),
239            PickleValue::Dict(dict) => {
240                let mut dict_iter = dict.into_iter();
241                let Some((inner_key, PickleValue::List(value))) = dict_iter.next() else { return Err(Error::PickleFormatError) };
242
243                Ok((format!("{} {}", key, inner_key), value))
244            }
245            _ => Err(Error::DecompressionError.into()),
246        })
247        .collect()
248}
wot_datfile_parser/parser.rs

wot_datfile_parser/
parser.rs
