bgpkit_parser/parser/
mod.rs

1/*!
2parser module maintains the main logic for processing BGP and MRT messages.
3*/
4use std::io::Read;
5
6#[macro_use]
7pub mod utils;
8pub mod bgp;
9pub mod bmp;
10pub mod filter;
11pub mod iters;
12pub mod mrt;
13pub mod rpki;
14
15#[cfg(feature = "rislive")]
16pub mod rislive;
17
18pub(crate) use self::utils::*;
19
20use crate::models::MrtRecord;
21pub use mrt::mrt_elem::{BgpUpdateElemIter, ElemError, Elementor, RecordElemIter};
22#[cfg(feature = "oneio")]
23use oneio::{get_cache_reader, get_reader};
24
25pub use crate::error::{ParserError, ParserErrorWithBytes};
26pub use bmp::{parse_bmp_msg, parse_openbmp_header, parse_openbmp_msg};
27pub use filter::*;
28pub use iters::*;
29pub use mrt::*;
30
31#[cfg(feature = "rislive")]
32pub use rislive::messages::{
33    RisLiveClientMessage, RisSubscribe, RisSubscribeSocketOptions, RisSubscribeType,
34};
35#[cfg(feature = "rislive")]
36pub use rislive::{
37    parse_ris_live_message, parse_ris_live_message_json, parse_ris_live_message_raw,
38};
39
40pub struct BgpkitParser<R> {
41    reader: R,
42    core_dump: bool,
43    filters: Vec<Filter>,
44    options: ParserOptions,
45}
46
47pub(crate) struct ParserOptions {
48    show_warnings: bool,
49}
50impl Default for ParserOptions {
51    fn default() -> Self {
52        ParserOptions {
53            show_warnings: true,
54        }
55    }
56}
57
58#[cfg(feature = "oneio")]
59impl BgpkitParser<Box<dyn Read + Send>> {
60    /// Creating a new parser from a object that implements [Read] trait.
61    pub fn new(path: &str) -> Result<Self, ParserErrorWithBytes> {
62        let reader = get_reader(path)?;
63        Ok(BgpkitParser {
64            reader,
65            core_dump: false,
66            filters: vec![],
67            options: ParserOptions::default(),
68        })
69    }
70
71    /// Creating a new parser that also caches the remote content to a local cache directory.
72    ///
73    /// The cache file name is generated by the following format: `cache-<crc32 of file name>-<file name>`.
74    /// For example, the remote file `http://archive.routeviews.org/route-views.chile/bgpdata/2023.03/RIBS/rib.20230326.0600.bz2`
75    /// will be cached as `cache-682cb1eb-rib.20230326.0600.bz2` in the cache directory.
76    pub fn new_cached(path: &str, cache_dir: &str) -> Result<Self, ParserErrorWithBytes> {
77        let file_name = path.rsplit('/').next().unwrap().to_string();
78        let new_file_name = format!(
79            "cache-{}",
80            add_suffix_to_filename(file_name.as_str(), crc32(path).as_str())
81        );
82        let reader = get_cache_reader(path, cache_dir, Some(new_file_name), false)?;
83        Ok(BgpkitParser {
84            reader,
85            core_dump: false,
86            filters: vec![],
87            options: ParserOptions::default(),
88        })
89    }
90}
91
92#[cfg(feature = "oneio")]
93fn add_suffix_to_filename(filename: &str, suffix: &str) -> String {
94    let mut parts: Vec<&str> = filename.split('.').collect(); // Split filename by dots
95    if parts.len() > 1 {
96        let last_part = parts.pop().unwrap(); // Remove the last part (suffix) from the parts vector
97        let new_last_part = format!("{suffix}.{last_part}"); // Add the suffix to the last part
98        parts.push(&new_last_part); // Add the updated last part back to the parts vector
99        parts.join(".") // Join the parts back into a filename string with dots
100    } else {
101        // If the filename does not have any dots, simply append the suffix to the end
102        format!("{filename}.{suffix}")
103    }
104}
105
106impl<R: Read> BgpkitParser<R> {
107    /// Creating a new parser from an object that implements [Read] trait.
108    pub fn from_reader(reader: R) -> Self {
109        BgpkitParser {
110            reader,
111            core_dump: false,
112            filters: vec![],
113            options: ParserOptions::default(),
114        }
115    }
116
117    /// This is used in for loop `for item in parser{}`
118    pub fn next_record(&mut self) -> Result<MrtRecord, ParserErrorWithBytes> {
119        parse_mrt_record(&mut self.reader)
120    }
121}
122
123impl<R> BgpkitParser<R> {
124    pub fn enable_core_dump(self) -> Self {
125        BgpkitParser {
126            reader: self.reader,
127            core_dump: true,
128            filters: self.filters,
129            options: self.options,
130        }
131    }
132
133    pub fn disable_warnings(self) -> Self {
134        let mut options = self.options;
135        options.show_warnings = false;
136        BgpkitParser {
137            reader: self.reader,
138            core_dump: self.core_dump,
139            filters: self.filters,
140            options,
141        }
142    }
143
144    /// Add a filter to the parser by specifying filter type and value as strings.
145    ///
146    /// This method parses the filter type and value strings to create a [`Filter`] and adds it
147    /// to the parser's filter list. For the full list of available filter types and their
148    /// formats, see the [`Filter`] struct documentation.
149    ///
150    /// # Available Filter Types
151    ///
152    /// - `origin_asn` - Origin AS number (e.g., "12345")
153    /// - `origin_asns` - Multiple origin AS numbers, comma-separated (e.g., "12345,67890")
154    /// - `prefix` - Exact prefix match (e.g., "192.168.1.0/24")
155    /// - `prefix_super` - Match prefix and super-prefixes
156    /// - `prefix_sub` - Match prefix and sub-prefixes
157    /// - `prefix_super_sub` - Match prefix, super-prefixes, and sub-prefixes
158    /// - `prefixes` - Multiple prefixes (e.g., "1.1.1.0/24,8.8.8.0/24")
159    /// - `peer_ip` - Peer IP address (e.g., "192.168.1.1")
160    /// - `peer_ips` - Multiple peer IPs (e.g., "192.168.1.1,192.168.1.2")
161    /// - `peer_asn` - Peer AS number (e.g., "12345")
162    /// - `peer_asns` - Multiple peer AS numbers (e.g., "12345,67890")
163    /// - `type` - Message type: "a"/"announce" or "w"/"withdraw"
164    /// - `ts_start` - Start timestamp (unix timestamp or RFC3339)
165    /// - `ts_end` - End timestamp (unix timestamp or RFC3339)
166    /// - `as_path` - AS path regex pattern
167    /// - `community` - Community regex pattern
168    /// - `ip_version` - IP version: "4"/"ipv4" or "6"/"ipv6"
169    ///
170    /// # Negative Filters
171    ///
172    /// Most filters support negation by prefixing the value with `!`. For example:
173    /// - `origin_asn=!13335` matches elements where origin AS is NOT 13335
174    /// - `prefix=!10.0.0.0/8` matches elements where prefix is NOT 10.0.0.0/8
175    ///
176    /// # Example
177    ///
178    /// ```no_run
179    /// use bgpkit_parser::BgpkitParser;
180    ///
181    /// let parser = BgpkitParser::new("https://spaces.bgpkit.org/parser/update-example.gz")
182    ///     .unwrap()
183    ///     .add_filter("peer_ip", "185.1.8.65")
184    ///     .unwrap()
185    ///     .add_filter("type", "w")
186    ///     .unwrap();
187    ///
188    /// for elem in parser {
189    ///     println!("{}", elem);
190    /// }
191    /// ```
192    pub fn add_filter(
193        self,
194        filter_type: &str,
195        filter_value: &str,
196    ) -> Result<Self, ParserErrorWithBytes> {
197        let mut filters = self.filters;
198        filters.push(Filter::new(filter_type, filter_value)?);
199        Ok(BgpkitParser {
200            reader: self.reader,
201            core_dump: self.core_dump,
202            filters,
203            options: self.options,
204        })
205    }
206
207    /// Add multiple filters to the parser.
208    ///
209    /// This method extends the existing filters with the provided slice of filters.
210    ///
211    /// # Example
212    ///
213    /// ```no_run
214    /// use bgpkit_parser::BgpkitParser;
215    /// use bgpkit_parser::parser::Filter;
216    ///
217    /// let filters = vec![
218    ///     Filter::new("peer_ip", "185.1.8.65").unwrap(),
219    ///     Filter::new("type", "w").unwrap(),
220    /// ];
221    ///
222    /// let parser = BgpkitParser::new("https://spaces.bgpkit.org/parser/update-example.gz")
223    ///     .unwrap()
224    ///     .add_filters(&filters);
225    /// ```
226    pub fn add_filters(mut self, filters: &[Filter]) -> Self {
227        self.filters.extend(filters.iter().cloned());
228        self
229    }
230
231    /// Set filters directly, replacing any existing filters.
232    ///
233    /// This method allows passing a pre-built `Vec<Filter>` directly to the parser,
234    /// bypassing the need to parse filter strings. This is useful when you want to
235    /// build filter specifications independently and reuse them across multiple parsers.
236    ///
237    /// # Example
238    ///
239    /// ```no_run
240    /// use bgpkit_parser::BgpkitParser;
241    /// use bgpkit_parser::parser::Filter;
242    ///
243    /// // Build filters independently
244    /// let filters = vec![
245    ///     Filter::new("peer_ip", "185.1.8.65").unwrap(),
246    ///     Filter::new("type", "w").unwrap(),
247    /// ];
248    ///
249    /// // Apply to multiple parsers (no manual clone needed)
250    /// let parser1 = BgpkitParser::new("https://spaces.bgpkit.org/parser/update-example.gz")
251    ///     .unwrap()
252    ///     .with_filters(&filters);
253    ///
254    /// let parser2 = BgpkitParser::new("https://spaces.bgpkit.org/parser/update-example.gz")
255    ///     .unwrap()
256    ///     .with_filters(&filters);
257    /// ```
258    pub fn with_filters(mut self, filters: &[Filter]) -> Self {
259        self.filters = filters.to_vec();
260        self
261    }
262}
263
264#[cfg(test)]
265mod tests {
266    use super::*;
267
268    #[test]
269    fn test_new_with_reader() {
270        // bzip2 reader for a compressed file
271        let reader = oneio::get_reader("http://archive.routeviews.org/route-views.ny/bgpdata/2023.02/UPDATES/updates.20230215.0630.bz2").unwrap();
272        assert_eq!(
273            12683,
274            BgpkitParser::from_reader(reader).into_elem_iter().count()
275        );
276
277        // remote reader for an uncompressed updates file
278        let reader = oneio::get_reader("https://spaces.bgpkit.org/parser/update-example").unwrap();
279        assert_eq!(
280            8160,
281            BgpkitParser::from_reader(reader).into_elem_iter().count()
282        );
283    }
284
285    #[test]
286    fn test_new_cached_with_reader() {
287        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
288        let parser = BgpkitParser::new_cached(url, "/tmp/bgpkit-parser-tests")
289            .unwrap()
290            .enable_core_dump()
291            .disable_warnings();
292        let count = parser.into_elem_iter().count();
293        assert_eq!(8160, count);
294        let parser = BgpkitParser::new_cached(url, "/tmp/bgpkit-parser-tests").unwrap();
295        let count = parser.into_elem_iter().count();
296        assert_eq!(8160, count);
297    }
298
299    #[test]
300    fn test_add_suffix_to_filename() {
301        // Test with a filename that has dots
302        let filename = "example.txt";
303        let suffix = "suffix";
304        let result = add_suffix_to_filename(filename, suffix);
305        assert_eq!(result, "example.suffix.txt");
306
307        // Test with a filename that has multiple dots
308        let filename = "example.tar.gz";
309        let suffix = "suffix";
310        let result = add_suffix_to_filename(filename, suffix);
311        assert_eq!(result, "example.tar.suffix.gz");
312
313        // Test with a filename that has no dots
314        let filename = "example";
315        let suffix = "suffix";
316        let result = add_suffix_to_filename(filename, suffix);
317        assert_eq!(result, "example.suffix");
318
319        // Test with an empty filename
320        let filename = "";
321        let suffix = "suffix";
322        let result = add_suffix_to_filename(filename, suffix);
323        assert_eq!(result, ".suffix");
324
325        // Test with an empty suffix
326        let filename = "example.txt";
327        let suffix = "";
328        let result = add_suffix_to_filename(filename, suffix);
329        assert_eq!(result, "example..txt");
330    }
331
332    #[test]
333    fn test_with_filters() {
334        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
335
336        // Build filters independently
337        let filters = vec![
338            Filter::new("peer_ip", "185.1.8.65").unwrap(),
339            Filter::new("type", "w").unwrap(),
340        ];
341
342        // Test with_filters - sets filters directly
343        let parser = BgpkitParser::new(url).unwrap().with_filters(&filters);
344        let count = parser.into_elem_iter().count();
345
346        // peer 185.1.8.65 has 3393 total, 132 withdrawals
347        assert_eq!(count, 132);
348
349        // Test that with_filters replaces existing filters
350        let filters1 = vec![Filter::new("peer_ip", "185.1.8.65").unwrap()];
351        let filters2 = vec![Filter::new("peer_ip", "185.1.8.50").unwrap()];
352
353        let parser = BgpkitParser::new(url)
354            .unwrap()
355            .with_filters(&filters1)
356            .with_filters(&filters2); // Should replace filters1
357        let count = parser.into_elem_iter().count();
358
359        // peer 185.1.8.50 has 1563 elements
360        assert_eq!(count, 1563);
361    }
362
363    #[test]
364    fn test_add_filters() {
365        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
366
367        // Build filters independently
368        let filters = vec![
369            Filter::new("peer_ip", "185.1.8.65").unwrap(),
370            Filter::new("type", "w").unwrap(),
371        ];
372
373        // Test add_filters - extends existing filters
374        let parser = BgpkitParser::new(url).unwrap().add_filters(&filters);
375        let count = parser.into_elem_iter().count();
376
377        // peer 185.1.8.65 has 3393 total, 132 withdrawals
378        assert_eq!(count, 132);
379
380        // Test combining add_filter and add_filters
381        let parser = BgpkitParser::new(url)
382            .unwrap()
383            .add_filter("peer_ip", "185.1.8.65")
384            .unwrap()
385            .add_filters(&[Filter::new("type", "w").unwrap()]);
386        let count = parser.into_elem_iter().count();
387        assert_eq!(count, 132);
388    }
389
390    #[test]
391    fn test_with_filters_empty() {
392        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
393
394        // Test with empty filters - should return all elements
395        let parser = BgpkitParser::new(url).unwrap().with_filters(&[]);
396        let count = parser.into_elem_iter().count();
397
398        // Total elements in the file
399        assert_eq!(count, 8160);
400    }
401
402    #[test]
403    fn test_add_filters_empty() {
404        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
405
406        // Test adding empty filters - should not change behavior
407        let parser = BgpkitParser::new(url)
408            .unwrap()
409            .add_filter("peer_ip", "185.1.8.65")
410            .unwrap()
411            .add_filters(&[]);
412        let count = parser.into_elem_iter().count();
413
414        // peer 185.1.8.65 has 3393 elements
415        assert_eq!(count, 3393);
416    }
417
418    #[test]
419    fn test_with_filters_reuse() {
420        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
421
422        // Build filters once
423        let filters = vec![
424            Filter::new("peer_ip", "185.1.8.65").unwrap(),
425            Filter::new("type", "w").unwrap(),
426        ];
427
428        // Apply to multiple parsers (simulating reuse pattern - no clone needed)
429        let parser1 = BgpkitParser::new(url).unwrap().with_filters(&filters);
430        let count1 = parser1.into_elem_iter().count();
431
432        let parser2 = BgpkitParser::new(url).unwrap().with_filters(&filters);
433        let count2 = parser2.into_elem_iter().count();
434
435        // Both should have same count: 132 withdrawals from peer 185.1.8.65
436        assert_eq!(count1, 132);
437        assert_eq!(count2, 132);
438    }
439}
bgpkit_parser/parser/mod.rs

bgpkit_parser/parser/
mod.rs
