bgpkit_parser/parser/
mod.rs

1/*!
2parser module maintains the main logic for processing BGP and MRT messages.
3*/
4use std::io::Read;
5
6#[macro_use]
7pub mod utils;
8pub mod bgp;
9pub mod bmp;
10pub mod filter;
11pub mod iters;
12pub mod mrt;
13pub mod rpki;
14
15#[cfg(feature = "rislive")]
16pub mod rislive;
17
18pub(crate) use self::utils::*;
19
20use crate::models::MrtRecord;
21pub use mrt::mrt_elem::{BgpUpdateElemIter, ElemError, Elementor, RecordElemIter};
22#[cfg(feature = "oneio")]
23use oneio::{get_cache_reader, get_reader};
24
25pub use crate::error::{ParserError, ParserErrorWithBytes};
26pub use bmp::{parse_bmp_msg, parse_openbmp_header, parse_openbmp_msg};
27pub use filter::*;
28pub use iters::*;
29pub use mrt::*;
30
31#[cfg(feature = "rislive")]
32pub use rislive::messages::{
33    RisLiveClientMessage, RisSubscribe, RisSubscribeSocketOptions, RisSubscribeType,
34};
35#[cfg(feature = "rislive")]
36pub use rislive::{
37    parse_ris_live_message, parse_ris_live_message_json, parse_ris_live_message_raw,
38};
39
40pub struct BgpkitParser<R> {
41    reader: R,
42    core_dump: bool,
43    filters: Vec<Filter>,
44    options: ParserOptions,
45}
46
47pub(crate) struct ParserOptions {
48    show_warnings: bool,
49}
50impl Default for ParserOptions {
51    fn default() -> Self {
52        ParserOptions {
53            show_warnings: true,
54        }
55    }
56}
57
58#[cfg(feature = "oneio")]
59impl BgpkitParser<Box<dyn Read + Send>> {
60    /// Creating a new parser from a object that implements [Read] trait.
61    pub fn new(path: &str) -> Result<Self, ParserErrorWithBytes> {
62        let reader = get_reader(path)?;
63        Ok(BgpkitParser {
64            reader,
65            core_dump: false,
66            filters: vec![],
67            options: ParserOptions::default(),
68        })
69    }
70
71    /// Creating a new parser that also caches the remote content to a local cache directory.
72    ///
73    /// The cache file name is generated by the following format: `cache-<crc32 of file name>-<file name>`.
74    /// For example, the remote file `http://archive.routeviews.org/route-views.chile/bgpdata/2023.03/RIBS/rib.20230326.0600.bz2`
75    /// will be cached as `cache-682cb1eb-rib.20230326.0600.bz2` in the cache directory.
76    pub fn new_cached(path: &str, cache_dir: &str) -> Result<Self, ParserErrorWithBytes> {
77        let file_name = path.rsplit('/').next().unwrap().to_string();
78        let new_file_name = format!(
79            "cache-{}",
80            add_suffix_to_filename(file_name.as_str(), crc32(path).as_str())
81        );
82        let reader = get_cache_reader(path, cache_dir, Some(new_file_name), false)?;
83        Ok(BgpkitParser {
84            reader,
85            core_dump: false,
86            filters: vec![],
87            options: ParserOptions::default(),
88        })
89    }
90}
91
92#[cfg(feature = "oneio")]
93fn add_suffix_to_filename(filename: &str, suffix: &str) -> String {
94    let mut parts: Vec<&str> = filename.split('.').collect(); // Split filename by dots
95    if parts.len() > 1 {
96        let last_part = parts.pop().unwrap(); // Remove the last part (suffix) from the parts vector
97        let new_last_part = format!("{suffix}.{last_part}"); // Add the suffix to the last part
98        parts.push(&new_last_part); // Add the updated last part back to the parts vector
99        parts.join(".") // Join the parts back into a filename string with dots
100    } else {
101        // If the filename does not have any dots, simply append the suffix to the end
102        format!("{filename}.{suffix}")
103    }
104}
105
106impl<R: Read> BgpkitParser<R> {
107    /// Creating a new parser from an object that implements [Read] trait.
108    pub fn from_reader(reader: R) -> Self {
109        BgpkitParser {
110            reader,
111            core_dump: false,
112            filters: vec![],
113            options: ParserOptions::default(),
114        }
115    }
116
117    /// This is used in for loop `for item in parser{}`
118    pub fn next_record(&mut self) -> Result<MrtRecord, ParserErrorWithBytes> {
119        parse_mrt_record(&mut self.reader)
120    }
121}
122
123impl<R> BgpkitParser<R> {
124    pub fn enable_core_dump(self) -> Self {
125        BgpkitParser {
126            reader: self.reader,
127            core_dump: true,
128            filters: self.filters,
129            options: self.options,
130        }
131    }
132
133    pub fn disable_warnings(self) -> Self {
134        let mut options = self.options;
135        options.show_warnings = false;
136        BgpkitParser {
137            reader: self.reader,
138            core_dump: self.core_dump,
139            filters: self.filters,
140            options,
141        }
142    }
143
144    /// Add a filter to the parser by specifying filter type and value as strings.
145    ///
146    /// This method parses the filter type and value strings to create a [`Filter`] and adds it
147    /// to the parser's filter list. For the full list of available filter types and their
148    /// formats, see the [`Filter`] struct documentation.
149    ///
150    /// # Available Filter Types
151    ///
152    /// - `origin_asn` - Origin AS number (e.g., "12345")
153    /// - `origin_asns` - Multiple origin AS numbers, comma-separated (e.g., "12345,67890")
154    /// - `prefix` - Exact prefix match (e.g., "192.168.1.0/24")
155    /// - `prefix_super` - Match prefix and super-prefixes
156    /// - `prefix_sub` - Match prefix and sub-prefixes
157    /// - `prefix_super_sub` - Match prefix, super-prefixes, and sub-prefixes
158    /// - `prefixes` - Multiple prefixes (e.g., "1.1.1.0/24,8.8.8.0/24")
159    /// - `peer_ip` - Peer IP address (e.g., "192.168.1.1")
160    /// - `peer_ips` - Multiple peer IPs (e.g., "192.168.1.1,192.168.1.2")
161    /// - `peer_asn` - Peer AS number (e.g., "12345")
162    /// - `peer_asns` - Multiple peer AS numbers (e.g., "12345,67890")
163    /// - `type` - Message type: "a"/"announce" or "w"/"withdraw"
164    /// - `ts_start` - Start timestamp (unix timestamp or RFC3339)
165    /// - `ts_end` - End timestamp (unix timestamp or RFC3339)
166    /// - `as_path` - AS path regex pattern
167    /// - `community` - Community regex pattern
168    /// - `ip_version` - IP version: "4"/"ipv4" or "6"/"ipv6"
169    /// - `otc` - Only-to-customer ASN (RFC 9234); `*` for present, `!*` for absent
170    /// - `next_hop` - Next hop IP address; `*`/`!*` for presence
171    /// - `origin` - Origin attribute: "igp", "egp", or "incomplete"; `*`/`!*` for presence
172    /// - `local_pref` - Local preference value; `*`/`!*` for presence
173    /// - `med` - Multi-exit discriminator value; `*`/`!*` for presence
174    /// - `atomic` - Atomic aggregate flag: "true"/"false"
175    /// - `aggr_asn` - Aggregator ASN; `*`/`!*` for presence
176    /// - `aggr_ip` - Aggregator IP address; `*`/`!*` for presence
177    /// - `peer_bgp_id` - Peer BGP identifier (router ID); `*`/`!*` for presence
178    ///
179    /// # Negative Filters
180    ///
181    /// Most filters support negation by prefixing the value with `!`. For example:
182    /// - `origin_asn=!13335` matches elements where origin AS is NOT 13335
183    /// - `prefix=!10.0.0.0/8` matches elements where prefix is NOT 10.0.0.0/8
184    ///
185    /// # Presence Filters
186    ///
187    /// Optional fields (`Option<T>`) support `*` as a wildcard to check whether a field
188    /// is present or absent: `otc=*` matches elements with an OTC value, `otc=!*` matches
189    /// elements without one.
190    ///
191    /// # Example
192    ///
193    /// ```no_run
194    /// use bgpkit_parser::BgpkitParser;
195    ///
196    /// let parser = BgpkitParser::new("https://spaces.bgpkit.org/parser/update-example.gz")
197    ///     .unwrap()
198    ///     .add_filter("peer_ip", "185.1.8.65")
199    ///     .unwrap()
200    ///     .add_filter("type", "w")
201    ///     .unwrap();
202    ///
203    /// for elem in parser {
204    ///     println!("{}", elem);
205    /// }
206    /// ```
207    pub fn add_filter(
208        self,
209        filter_type: &str,
210        filter_value: &str,
211    ) -> Result<Self, ParserErrorWithBytes> {
212        let mut filters = self.filters;
213        filters.push(Filter::new(filter_type, filter_value)?);
214        Ok(BgpkitParser {
215            reader: self.reader,
216            core_dump: self.core_dump,
217            filters,
218            options: self.options,
219        })
220    }
221
222    /// Add multiple filters to the parser.
223    ///
224    /// This method extends the existing filters with the provided slice of filters.
225    ///
226    /// # Example
227    ///
228    /// ```no_run
229    /// use bgpkit_parser::BgpkitParser;
230    /// use bgpkit_parser::parser::Filter;
231    ///
232    /// let filters = vec![
233    ///     Filter::new("peer_ip", "185.1.8.65").unwrap(),
234    ///     Filter::new("type", "w").unwrap(),
235    /// ];
236    ///
237    /// let parser = BgpkitParser::new("https://spaces.bgpkit.org/parser/update-example.gz")
238    ///     .unwrap()
239    ///     .add_filters(&filters);
240    /// ```
241    pub fn add_filters(mut self, filters: &[Filter]) -> Self {
242        self.filters.extend(filters.iter().cloned());
243        self
244    }
245
246    /// Set filters directly, replacing any existing filters.
247    ///
248    /// This method allows passing a pre-built `Vec<Filter>` directly to the parser,
249    /// bypassing the need to parse filter strings. This is useful when you want to
250    /// build filter specifications independently and reuse them across multiple parsers.
251    ///
252    /// # Example
253    ///
254    /// ```no_run
255    /// use bgpkit_parser::BgpkitParser;
256    /// use bgpkit_parser::parser::Filter;
257    ///
258    /// // Build filters independently
259    /// let filters = vec![
260    ///     Filter::new("peer_ip", "185.1.8.65").unwrap(),
261    ///     Filter::new("type", "w").unwrap(),
262    /// ];
263    ///
264    /// // Apply to multiple parsers (no manual clone needed)
265    /// let parser1 = BgpkitParser::new("https://spaces.bgpkit.org/parser/update-example.gz")
266    ///     .unwrap()
267    ///     .with_filters(&filters);
268    ///
269    /// let parser2 = BgpkitParser::new("https://spaces.bgpkit.org/parser/update-example.gz")
270    ///     .unwrap()
271    ///     .with_filters(&filters);
272    /// ```
273    pub fn with_filters(mut self, filters: &[Filter]) -> Self {
274        self.filters = filters.to_vec();
275        self
276    }
277}
278
279#[cfg(test)]
280mod tests {
281    use super::*;
282
283    #[test]
284    fn test_new_with_reader() {
285        // bzip2 reader for a compressed file
286        let reader = oneio::get_reader("http://archive.routeviews.org/route-views.ny/bgpdata/2023.02/UPDATES/updates.20230215.0630.bz2").unwrap();
287        assert_eq!(
288            12683,
289            BgpkitParser::from_reader(reader).into_elem_iter().count()
290        );
291
292        // remote reader for an uncompressed updates file
293        let reader = oneio::get_reader("https://spaces.bgpkit.org/parser/update-example").unwrap();
294        assert_eq!(
295            8160,
296            BgpkitParser::from_reader(reader).into_elem_iter().count()
297        );
298    }
299
300    #[test]
301    fn test_new_cached_with_reader() {
302        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
303        let parser = BgpkitParser::new_cached(url, "/tmp/bgpkit-parser-tests")
304            .unwrap()
305            .enable_core_dump()
306            .disable_warnings();
307        let count = parser.into_elem_iter().count();
308        assert_eq!(8160, count);
309        let parser = BgpkitParser::new_cached(url, "/tmp/bgpkit-parser-tests").unwrap();
310        let count = parser.into_elem_iter().count();
311        assert_eq!(8160, count);
312    }
313
314    #[test]
315    fn test_add_suffix_to_filename() {
316        // Test with a filename that has dots
317        let filename = "example.txt";
318        let suffix = "suffix";
319        let result = add_suffix_to_filename(filename, suffix);
320        assert_eq!(result, "example.suffix.txt");
321
322        // Test with a filename that has multiple dots
323        let filename = "example.tar.gz";
324        let suffix = "suffix";
325        let result = add_suffix_to_filename(filename, suffix);
326        assert_eq!(result, "example.tar.suffix.gz");
327
328        // Test with a filename that has no dots
329        let filename = "example";
330        let suffix = "suffix";
331        let result = add_suffix_to_filename(filename, suffix);
332        assert_eq!(result, "example.suffix");
333
334        // Test with an empty filename
335        let filename = "";
336        let suffix = "suffix";
337        let result = add_suffix_to_filename(filename, suffix);
338        assert_eq!(result, ".suffix");
339
340        // Test with an empty suffix
341        let filename = "example.txt";
342        let suffix = "";
343        let result = add_suffix_to_filename(filename, suffix);
344        assert_eq!(result, "example..txt");
345    }
346
347    #[test]
348    fn test_with_filters() {
349        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
350
351        // Build filters independently
352        let filters = vec![
353            Filter::new("peer_ip", "185.1.8.65").unwrap(),
354            Filter::new("type", "w").unwrap(),
355        ];
356
357        // Test with_filters - sets filters directly
358        let parser = BgpkitParser::new(url).unwrap().with_filters(&filters);
359        let count = parser.into_elem_iter().count();
360
361        // peer 185.1.8.65 has 3393 total, 132 withdrawals
362        assert_eq!(count, 132);
363
364        // Test that with_filters replaces existing filters
365        let filters1 = vec![Filter::new("peer_ip", "185.1.8.65").unwrap()];
366        let filters2 = vec![Filter::new("peer_ip", "185.1.8.50").unwrap()];
367
368        let parser = BgpkitParser::new(url)
369            .unwrap()
370            .with_filters(&filters1)
371            .with_filters(&filters2); // Should replace filters1
372        let count = parser.into_elem_iter().count();
373
374        // peer 185.1.8.50 has 1563 elements
375        assert_eq!(count, 1563);
376    }
377
378    #[test]
379    fn test_add_filters() {
380        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
381
382        // Build filters independently
383        let filters = vec![
384            Filter::new("peer_ip", "185.1.8.65").unwrap(),
385            Filter::new("type", "w").unwrap(),
386        ];
387
388        // Test add_filters - extends existing filters
389        let parser = BgpkitParser::new(url).unwrap().add_filters(&filters);
390        let count = parser.into_elem_iter().count();
391
392        // peer 185.1.8.65 has 3393 total, 132 withdrawals
393        assert_eq!(count, 132);
394
395        // Test combining add_filter and add_filters
396        let parser = BgpkitParser::new(url)
397            .unwrap()
398            .add_filter("peer_ip", "185.1.8.65")
399            .unwrap()
400            .add_filters(&[Filter::new("type", "w").unwrap()]);
401        let count = parser.into_elem_iter().count();
402        assert_eq!(count, 132);
403    }
404
405    #[test]
406    fn test_with_filters_empty() {
407        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
408
409        // Test with empty filters - should return all elements
410        let parser = BgpkitParser::new(url).unwrap().with_filters(&[]);
411        let count = parser.into_elem_iter().count();
412
413        // Total elements in the file
414        assert_eq!(count, 8160);
415    }
416
417    #[test]
418    fn test_add_filters_empty() {
419        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
420
421        // Test adding empty filters - should not change behavior
422        let parser = BgpkitParser::new(url)
423            .unwrap()
424            .add_filter("peer_ip", "185.1.8.65")
425            .unwrap()
426            .add_filters(&[]);
427        let count = parser.into_elem_iter().count();
428
429        // peer 185.1.8.65 has 3393 elements
430        assert_eq!(count, 3393);
431    }
432
433    #[test]
434    fn test_with_filters_reuse() {
435        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
436
437        // Build filters once
438        let filters = vec![
439            Filter::new("peer_ip", "185.1.8.65").unwrap(),
440            Filter::new("type", "w").unwrap(),
441        ];
442
443        // Apply to multiple parsers (simulating reuse pattern - no clone needed)
444        let parser1 = BgpkitParser::new(url).unwrap().with_filters(&filters);
445        let count1 = parser1.into_elem_iter().count();
446
447        let parser2 = BgpkitParser::new(url).unwrap().with_filters(&filters);
448        let count2 = parser2.into_elem_iter().count();
449
450        // Both should have same count: 132 withdrawals from peer 185.1.8.65
451        assert_eq!(count1, 132);
452        assert_eq!(count2, 132);
453    }
454}
bgpkit_parser/parser/mod.rs

bgpkit_parser/parser/
mod.rs
