1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
//! # IPFix
//!
//! References:
//! - <https://datatracker.ietf.org/doc/html/rfc7011>
//! - <https://en.wikipedia.org/wiki/IP_Flow_Information_Export>
//! - <https://www.ibm.com/docs/en/npi/1.3.1?topic=overview-ipfix-message-format>
//! - <https://www.iana.org/assignments/ipfix/ipfix.xhtml>

use super::data_number::*;
use crate::variable_versions::ipfix_lookup::*;
use crate::{NetflowPacket, NetflowParseError, ParsedNetflow};

use nom::bytes::complete::take;
use nom::error::{Error as NomError, ErrorKind};
use nom::multi::count;
use nom::Err as NomErr;
use nom::IResult;
use nom_derive::*;
use serde::Serialize;
use Nom;

use std::collections::BTreeMap;

const TEMPLATE_ID: u16 = 2;
const OPTIONS_TEMPLATE_ID: u16 = 3;
const SET_MIN_RANGE: u16 = 255;

type TemplateId = u16;
type IPFixFieldPair = (IPFixField, FieldValue);

pub(crate) fn parse_netflow_ipfix(
    packet: &[u8],
    parser: &mut IPFixParser,
) -> Result<ParsedNetflow, NetflowParseError> {
    IPFix::parse(packet, parser)
        .map(|(remaining, ipfix)| ParsedNetflow::new(remaining, NetflowPacket::IPFix(ipfix)))
        .map_err(|e| NetflowParseError::IPFix(e.to_string()))
}

#[derive(Default, Debug)]
pub struct IPFixParser {
    pub templates: BTreeMap<TemplateId, Template>,
    pub options_templates: BTreeMap<TemplateId, OptionsTemplate>,
}

#[derive(Nom, Debug, PartialEq, Clone, Serialize)]
#[nom(ExtraArgs(parser: &mut IPFixParser))]
pub struct IPFix {
    /// IPFix Header
    pub header: Header,
    /// Sets
    #[nom(Parse = "{ |i| parse_sets(i, parser, header.length) }")]
    pub flowsets: Vec<FlowSet>,
}

#[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Nom)]
pub struct Header {
    /// Version of Flow Record format that is exported in this message. The value of this
    /// field is 0x000a for the current version, incrementing by one the version that is
    /// used in the NetFlow services export version 9
    #[nom(Value = "10")]
    pub version: u16,
    /// Total length of the IPFIX Message, which is measured in octets, including Message
    /// Header and Sets.
    pub length: u16,
    /// Time, in seconds, since 0000 Coordinated Universal Time Jan 1, 1970, at which the
    /// IPFIX Message Header leaves the Exporter.
    pub export_time: u32,
    /// Incremental sequence counter-modulo 2^32 of all IPFIX Data Records sent on this PR-SCTP
    /// stream from the current Observation Domain by the Exporting Process. Check the specific
    /// meaning of this field in the subsections of Section 10 when UDP or TCP is selected as the
    /// transport protocol. This value must be used by the Collecting Process to identify whether
    /// any IPFIX Data Records are missed. Template and Options Template Records do not increase
    /// the Sequence Number.
    pub sequence_number: u32,
    /// A 32-bit identifier of the Observation Domain that is locally unique to the Exporting Process.
    /// The Exporting Process uses the Observation Domain ID to uniquely identify to the Collector.
    /// Process the Observation Domain that metered the Flows. It is recommended that this identifier
    /// is unique per IPFIX Device. Collecting Processes must use the Transport Session.  Observation
    /// Domain ID field to separate different export streams that originate from the same Exporting Process.
    /// The Observation Domain ID must be 0 when no specific Observation Domain ID is relevant for the
    /// entire IPFIX Message. For example, when the Exporting Process Statistics are exported, or in a hierarchy
    /// of Collectors when aggregated Data Records are exported.
    pub observation_domain_id: u32,
}

#[derive(Debug, PartialEq, Clone, Serialize, Nom)]
#[nom(ExtraArgs(parser: &mut IPFixParser))]
pub struct FlowSet {
    pub header: FlowSetHeader,
    #[nom(Parse = "{ |i| parse_set_body(i, parser, header.length, header.header_id) }")]
    pub body: FlowSetBody,
}

#[derive(Debug, PartialEq, Clone, Serialize, Nom)]
pub struct FlowSetHeader {
    /// Set ID value identifies the Set. A value of 2 is reserved for the Template Set.
    /// A value of 3 is reserved for the Option Template Set. All other values 4-255 are
    /// reserved for future use. Values more than 255 are used for Data Sets. The Set ID
    /// values of 0 and 1 are not used for historical reasons
    pub header_id: u16,
    /// Total length of the Set, in octets, including the Set Header, all records, and the
    /// optional padding. Because an individual Set MAY contain multiple records, the Length
    /// value must be used to determine the position of the next Set.
    pub length: u16,
}

#[derive(Debug, PartialEq, Clone, Serialize, Nom)]
#[nom(ExtraArgs(parser: &mut IPFixParser, id: u16, length: u16))]
pub struct FlowSetBody {
    #[nom(
        Cond = "id == TEMPLATE_ID",
        // Save our templates
        PostExec = "if let Some(templates) = templates.clone() { parser.templates.insert(templates.template_id, templates); }"
    )]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub templates: Option<Template>,
    #[nom(
        Cond = "id == OPTIONS_TEMPLATE_ID",
        PreExec = "let set_length = length.checked_sub(4).unwrap_or(length);",
        Parse = "{ |i| OptionsTemplate::parse(i, set_length) }",
        // Save our templates
        PostExec = "if let Some(options_templates) = options_templates.clone() {
                      parser.options_templates.insert(options_templates.template_id, options_templates);
                    }"
    )]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub options_templates: Option<OptionsTemplate>,
    // Data
    #[nom(
        Cond = "id > SET_MIN_RANGE && parser.templates.contains_key(&id)",
        Parse = "{ |i| Data::parse(i, parser, id) }"
    )]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub data: Option<Data>,
    // OptionsData
    #[nom(
        Cond = "id > SET_MIN_RANGE && parser.options_templates.contains_key(&id)",
        Parse = "{ |i| OptionsData::parse(i, parser, id) }"
    )]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub options_data: Option<OptionsData>,
}

#[derive(Debug, PartialEq, Clone, Serialize, Nom)]
#[nom(ExtraArgs(parser: &mut IPFixParser, set_id: u16))]
pub struct Data {
    #[nom(Parse = "{ |i| parse_fields::<Template>(i, parser.templates.get(&set_id)) }")]
    pub data_fields: Vec<BTreeMap<usize, (IPFixField, FieldValue)>>,
}

#[derive(Debug, PartialEq, Clone, Serialize, Nom)]
#[nom(ExtraArgs(parser: &mut IPFixParser, set_id: u16))]
pub struct OptionsData {
    #[nom(
        Parse = "{ |i| parse_fields::<OptionsTemplate>(i, parser.options_templates.get(&set_id)) }"
    )]
    pub data_fields: Vec<BTreeMap<usize, (IPFixField, FieldValue)>>,
}

#[derive(Debug, PartialEq, Eq, Clone, Serialize, Nom)]
#[nom(ExtraArgs(set_length: u16))]
pub struct OptionsTemplate {
    pub template_id: u16,
    pub field_count: u16,
    pub scope_field_count: u16,
    #[nom(
        PreExec = "let combined_count = scope_field_count as usize + 
                       field_count.checked_sub(scope_field_count).unwrap_or(field_count) as usize;",
        Parse = "count(|i| TemplateField::parse(i, true), combined_count)",
        PostExec = "let options_remaining = set_length.checked_sub(field_count * 4).unwrap_or(set_length) > 0;"
    )]
    pub fields: Vec<TemplateField>,
    #[nom(Cond = "options_remaining && !i.is_empty()")]
    #[serde(skip_serializing)]
    padding: Option<u16>,
}

#[derive(Debug, PartialEq, Eq, Clone, Serialize, Nom)]
pub struct Template {
    pub template_id: u16,
    pub field_count: u16,
    #[nom(Parse = "{ |i| parse_template_fields(i, field_count) } ")]
    pub fields: Vec<TemplateField>,
}

fn parse_template_fields(i: &[u8], count: u16) -> IResult<&[u8], Vec<TemplateField>> {
    let mut result = vec![];

    let mut remaining = i;

    for _ in 0..count {
        let (i, field) = TemplateField::parse(remaining, false)?;
        result.push(field);
        remaining = i;
    }

    Ok((remaining, result))
}

#[derive(Debug, PartialEq, Eq, Clone, Serialize, Nom)]
#[nom(ExtraArgs(options_template: bool))]
pub struct TemplateField {
    pub field_type_number: u16,
    #[nom(Value(IPFixField::from(field_type_number)))]
    pub field_type: IPFixField,
    pub field_length: u16,
    #[nom(
        Cond = "options_template && field_type_number > 32767",
        PostExec = "let field_type_number = if options_template {
                      field_type_number.overflowing_sub(32768).0
                    } else { field_type_number };",
        PostExec = "let field_type = if options_template && enterprise_number.is_some() {
                        IPFixField::Enterprise
                    } else { field_type };"
    )]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub enterprise_number: Option<u32>,
}

// Common trait for both templates.  Mainly for fetching fields.
trait CommonTemplate {
    fn get_fields(&self) -> &Vec<TemplateField>;
}

impl CommonTemplate for Template {
    fn get_fields(&self) -> &Vec<TemplateField> {
        &self.fields
    }
}

impl CommonTemplate for OptionsTemplate {
    fn get_fields(&self) -> &Vec<TemplateField> {
        &self.fields
    }
}

// Custom parse set function to take only length provided by header.
fn parse_sets<'a>(
    i: &'a [u8],
    parser: &mut IPFixParser,
    length: u16,
) -> IResult<&'a [u8], Vec<FlowSet>> {
    let length = length.checked_sub(16).unwrap_or(length);
    let (_, taken) = take(length)(i)?;

    let mut sets = vec![];

    let mut remaining = taken;

    while !remaining.is_empty() {
        let (i, set) = FlowSet::parse(remaining, parser)?;
        sets.push(set);
        remaining = i;
    }

    Ok((remaining, sets))
}

// Custom parse set body function to take only length provided by set header.
fn parse_set_body<'a>(
    i: &'a [u8],
    parser: &mut IPFixParser,
    length: u16,
    id: u16,
) -> IResult<&'a [u8], FlowSetBody> {
    // length - 4 to account for the set header
    let length = length.checked_sub(4).unwrap_or(length);
    let (remaining, taken) = take(length)(i)?;
    let (_, set_body) = FlowSetBody::parse(taken, parser, id, length)?;
    Ok((remaining, set_body))
}

/// Takes a byte stream and a cached template.
/// Fields get matched to static types.
/// Returns BTree of IPFix Types & Fields or IResult Error.
fn parse_fields<'a, T: CommonTemplate>(
    i: &'a [u8],
    template: Option<&T>,
) -> IResult<&'a [u8], Vec<BTreeMap<usize, IPFixFieldPair>>> {
    fn parse_field<'a>(
        i: &'a [u8],
        template_field: &TemplateField,
    ) -> IResult<&'a [u8], FieldValue> {
        // Enterprise Number
        if template_field.enterprise_number.is_some() {
            let (remaining, data_number) = DataNumber::parse(i, 4, false)?;
            Ok((remaining, FieldValue::DataNumber(data_number)))
        // Type matching
        } else {
            Ok(DataNumber::from_field_type(
                i,
                template_field.field_type.into(),
                template_field.field_length,
            ))?
        }
    }

    // If no fields there are no fields to parse, return an error.
    let template_fields = template
        .ok_or(NomErr::Error(NomError::new(i, ErrorKind::Fail)))?
        .get_fields();

    if template_fields.is_empty() {
        // dbg!("Template without fields!");
        return Err(NomErr::Error(NomError::new(i, ErrorKind::Fail)));
    };

    let mut fields = vec![];
    let mut remaining = i;

    let total_size = template_fields
        .iter()
        .map(|m| m.field_length as usize)
        .sum::<usize>();

    if total_size == 0 {
        return Ok((&[], fields));
    }
    let count: usize = i.len() / total_size;

    let mut error = false;

    // Iter through template fields and push them to a vec.  If we encouter any zero length fields we return an error.
    for _ in 0..count {
        let mut data_field = BTreeMap::new();
        for (c, template_field) in template_fields.iter().enumerate() {
            // If field length is 0 we error
            let (i, field_value) = parse_field(remaining, template_field)?;
            // If we don't move forward for some reason we error
            if i.len() == remaining.len() {
                error = true;
                break;
            }
            remaining = i;
            data_field.insert(c, (template_field.field_type, field_value));
        }
        fields.push(data_field);
    }

    if error {
        Err(NomErr::Error(NomError::new(remaining, ErrorKind::Fail)))
    } else {
        Ok((&[], fields))
    }
}

impl IPFix {
    /// Convert the IPFix to a `Vec<u8>` of bytes in big-endian order for exporting
    pub fn to_be_bytes(&self) -> Vec<u8> {
        let mut result = vec![];

        result.extend_from_slice(&self.header.version.to_be_bytes());
        result.extend_from_slice(&self.header.length.to_be_bytes());
        result.extend_from_slice(&(self.header.export_time).to_be_bytes());
        result.extend_from_slice(&self.header.sequence_number.to_be_bytes());
        result.extend_from_slice(&self.header.observation_domain_id.to_be_bytes());

        for flow in &self.flowsets {
            result.extend_from_slice(&flow.header.header_id.to_be_bytes());
            result.extend_from_slice(&flow.header.length.to_be_bytes());

            let mut result_flowset = vec![];

            if let Some(template) = &flow.body.templates {
                result_flowset.extend_from_slice(&template.template_id.to_be_bytes());
                result_flowset.extend_from_slice(&template.field_count.to_be_bytes());

                for field in template.fields.iter() {
                    result_flowset.extend_from_slice(&field.field_type_number.to_be_bytes());
                    result_flowset.extend_from_slice(&field.field_length.to_be_bytes());
                    if let Some(enterprise) = field.enterprise_number {
                        result_flowset.extend_from_slice(&enterprise.to_be_bytes());
                    }
                }
            }

            if let Some(options_template) = &flow.body.options_templates {
                result_flowset.extend_from_slice(&options_template.template_id.to_be_bytes());
                result_flowset.extend_from_slice(&options_template.field_count.to_be_bytes());
                result_flowset
                    .extend_from_slice(&options_template.scope_field_count.to_be_bytes());

                for field in options_template.fields.iter() {
                    result_flowset.extend_from_slice(&field.field_type_number.to_be_bytes());
                    result_flowset.extend_from_slice(&field.field_length.to_be_bytes());
                    if let Some(enterprise) = field.enterprise_number {
                        result_flowset.extend_from_slice(&enterprise.to_be_bytes());
                    }
                }
                if let Some(padding) = &options_template.padding {
                    result_flowset.extend_from_slice(&padding.to_be_bytes());
                }
            }

            if let Some(data) = &flow.body.data {
                for item in data.data_fields.iter() {
                    for (_, (_, v)) in item.iter() {
                        result_flowset.extend_from_slice(&v.to_be_bytes());
                    }
                }
            }

            if let Some(data) = &flow.body.options_data {
                for item in data.data_fields.iter() {
                    for (_, (_, v)) in item.iter() {
                        result_flowset.extend_from_slice(&v.to_be_bytes());
                    }
                }
            }

            result.append(&mut result_flowset);
        }

        result
    }
}