bgpkit_parser/parser/
filter.rs

1/*!
2## Message Filters
3
4The filter module defines a number of available filters that users can use, and implements
5the filtering mechanism for [BgpElem].
6
7The available filters are:
8- `origin_asn` -- origin AS number
9- `prefix` -- network prefix and match type
10- `peer_ip` -- peer's IP address
11- `peer_ips` -- peers' IP addresses
12- `peer_asn` -- peer's IP address
13- `type` -- message type (`withdraw` or `announce`)
14- `ts_start` -- start and end unix timestamp
15- `as_path` -- regular expression for AS path string
16- `ip_version` -- IP version (`ipv4` or `ipv6`)
17
18### Negative Filters
19
20Most filters support negation by prefixing the filter type with `!`. For example:
21- `!origin_asn` -- matches elements where origin AS is NOT the specified value
22- `!prefix` -- matches elements where prefix is NOT the specified value
23- `!peer_ip` -- matches elements where peer IP is NOT the specified value
24
25**Note**: Timestamp filters (`ts_start`, `ts_end`) do not support negation as the behavior would be unintuitive.
26
27[Filter::new] function takes a `str` as the filter type and `str` as the filter value and returns a
28Result of a [Filter] or a parsing error.
29
30[BgpkitParser](crate::BgpkitParser) implements the function `add_filter("filter_type", "filter_value")` that takes the parser's ownership itself
31and returns a new parser with specified filter added. See the example below.
32
33### Example
34
35```no_run
36use bgpkit_parser::BgpkitParser;
37
38/// This example shows how to parse a MRT file and filter by prefix.
39env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
40
41log::info!("downloading updates file");
42let parser = BgpkitParser::new("http://archive.routeviews.org/bgpdata/2021.10/UPDATES/updates.20211001.0000.bz2").unwrap()
43    .add_filter("prefix", "211.98.251.0/24").unwrap()
44    .add_filter("type", "a").unwrap();
45
46// iterating through the parser. the iterator returns `BgpElem` one at a time.
47log::info!("parsing updates file");
48for elem in parser {
49    log::info!("{}", &elem);
50}
51log::info!("done");
52```
53
54### Example with Negative Filter
55
56```no_run
57use bgpkit_parser::BgpkitParser;
58
59// Filter out all elements from AS 13335 (Cloudflare)
60let parser = BgpkitParser::new("http://archive.routeviews.org/bgpdata/2021.10/UPDATES/updates.20211001.0000.bz2").unwrap()
61    .add_filter("!origin_asn", "13335").unwrap();
62
63for elem in parser {
64    println!("{}", elem);
65}
66```
67
68Note, by default, the prefix filtering is for the exact prefix. You can include super-prefixes or
69sub-prefixes when filtering by using `"prefix_super"`, `"prefix_sub"`, or  `"prefix_super_sub"` as
70the filter type string.
71
72### Note
73
74Currently, only [BgpElem] implements the filtering capability. Support for [MrtRecord] will come in
75later releases.
76
77*/
78use crate::models::*;
79use crate::parser::ComparableRegex;
80use crate::ParserError;
81use crate::ParserError::FilterError;
82use ipnet::IpNet;
83use std::net::IpAddr;
84use std::str::FromStr;
85
86/// Filter enum: definition of types of filters
87///
88/// The available filters are (`filter_type` (`FilterType`) -- definition):
89/// - `origin_asn` (`OriginAsn(u32)`) -- origin AS number
90/// - `prefix(_super, _sub, _super_sub)` (`Prefix(IpNet, PrefixMatchType)`) -- network prefix and match type
91/// - `peer_ip` (`PeerIp(IpAddr)`) -- peer's IP address
92/// - `peer_ips` (`Vec<PeerIp(IpAddr)>`) -- peers' IP addresses
93/// - `peer_asn` (`PeerAsn(u32)`) -- peer's IP address
94/// - `type` (`Type(ElemType)`) -- message type (`withdraw` or `announce`)
95/// - `ts_start` (`TsStart(f64)`) and `ts_end` (`TsEnd(f64)`) -- start and end unix timestamp
96/// - `as_path` (`ComparableRegex`) -- regular expression for AS path string
97/// - `community` (`ComparableRegex`) -- regular expression for community string
98/// - `ip_version` (`IpVersion`) -- IP version (`ipv4` or `ipv6`)
99///
100/// **Negative filters**: All filters support negation by prefixing the filter type with `!`.
101/// For example, `!origin_asn` matches elements where origin AS is NOT the specified value.
102/// This creates a `Negated(Box<Filter>)` variant that inverts the match result.
103#[derive(Debug, Clone, PartialEq)]
104pub enum Filter {
105    OriginAsn(u32),
106    Prefix(IpNet, PrefixMatchType),
107    PeerIp(IpAddr),
108    PeerIps(Vec<IpAddr>),
109    PeerAsn(u32),
110    Type(ElemType),
111    IpVersion(IpVersion),
112    TsStart(f64),
113    TsEnd(f64),
114    AsPath(ComparableRegex),
115    Community(ComparableRegex),
116    /// Negated filter - matches when the inner filter does NOT match
117    Negated(Box<Filter>),
118}
119
120#[derive(Debug, Clone, PartialEq, Eq)]
121pub enum IpVersion {
122    Ipv4,
123    Ipv6,
124}
125
126#[derive(Debug, Clone, PartialEq, Eq)]
127pub enum PrefixMatchType {
128    Exact,
129    IncludeSuper,
130    IncludeSub,
131    IncludeSuperSub,
132}
133
134fn parse_time_str(time_str: &str) -> Option<chrono::NaiveDateTime> {
135    if let Ok(t) = time_str.parse::<f64>() {
136        return chrono::DateTime::from_timestamp(t as i64, 0).map(|t| t.naive_utc());
137    }
138    if let Ok(t) = chrono::DateTime::parse_from_rfc3339(time_str) {
139        return Some(t.naive_utc());
140    }
141    None
142}
143
144impl Filter {
145    pub fn new(filter_type: &str, filter_value: &str) -> Result<Filter, ParserError> {
146        // Check for negation prefix
147        let (negated, actual_filter_type) = if let Some(stripped) = filter_type.strip_prefix('!') {
148            // Reject double negation (e.g., "!!origin_asn")
149            if stripped.starts_with('!') {
150                return Err(FilterError(format!(
151                    "invalid filter type '{}': double negation is not allowed",
152                    filter_type
153                )));
154            }
155            // Reject negation for timestamp filters (unintuitive behavior)
156            if stripped == "ts_start"
157                || stripped == "start_ts"
158                || stripped == "ts_end"
159                || stripped == "end_ts"
160            {
161                return Err(FilterError(format!(
162                    "invalid filter type '{}': timestamp filters do not support negation",
163                    filter_type
164                )));
165            }
166            (true, stripped)
167        } else {
168            (false, filter_type)
169        };
170
171        let base_filter = Self::new_base(actual_filter_type, filter_value)?;
172
173        if negated {
174            Ok(Filter::Negated(Box::new(base_filter)))
175        } else {
176            Ok(base_filter)
177        }
178    }
179
180    fn new_base(filter_type: &str, filter_value: &str) -> Result<Filter, ParserError> {
181        match filter_type {
182            "origin_asn" => match u32::from_str(filter_value) {
183                Ok(v) => Ok(Filter::OriginAsn(v)),
184                Err(_) => Err(FilterError(format!(
185                    "cannot parse origin asn from {filter_value}"
186                ))),
187            },
188            "prefix" => match IpNet::from_str(filter_value) {
189                Ok(v) => Ok(Filter::Prefix(v, PrefixMatchType::Exact)),
190                Err(_) => Err(FilterError(format!(
191                    "cannot parse prefix from {filter_value}"
192                ))),
193            },
194            "prefix_super" => match IpNet::from_str(filter_value) {
195                Ok(v) => Ok(Filter::Prefix(v, PrefixMatchType::IncludeSuper)),
196                Err(_) => Err(FilterError(format!(
197                    "cannot parse prefix from {filter_value}"
198                ))),
199            },
200            "prefix_sub" => match IpNet::from_str(filter_value) {
201                Ok(v) => Ok(Filter::Prefix(v, PrefixMatchType::IncludeSub)),
202                Err(_) => Err(FilterError(format!(
203                    "cannot parse prefix from {filter_value}"
204                ))),
205            },
206            "prefix_super_sub" => match IpNet::from_str(filter_value) {
207                Ok(v) => Ok(Filter::Prefix(v, PrefixMatchType::IncludeSuperSub)),
208                Err(_) => Err(FilterError(format!(
209                    "cannot parse prefix from {filter_value}"
210                ))),
211            },
212            "peer_ip" => match IpAddr::from_str(filter_value) {
213                Ok(v) => Ok(Filter::PeerIp(v)),
214                Err(_) => Err(FilterError(format!(
215                    "cannot parse peer IP from {filter_value}"
216                ))),
217            },
218            "peer_ips" => {
219                let mut ips = vec![];
220                for ip_str in filter_value.replace(' ', "").split(',') {
221                    match IpAddr::from_str(ip_str) {
222                        Ok(v) => ips.push(v),
223                        Err(_) => {
224                            return Err(FilterError(format!("cannot parse peer IP from {ip_str}")))
225                        }
226                    }
227                }
228                Ok(Filter::PeerIps(ips))
229            }
230            "peer_asn" => match u32::from_str(filter_value) {
231                Ok(v) => Ok(Filter::PeerAsn(v)),
232                Err(_) => Err(FilterError(format!(
233                    "cannot parse peer asn from {filter_value}"
234                ))),
235            },
236            "type" => match filter_value {
237                "w" | "withdraw" | "withdrawal" => Ok(Filter::Type(ElemType::WITHDRAW)),
238                "a" | "announce" | "announcement" => Ok(Filter::Type(ElemType::ANNOUNCE)),
239                _ => Err(FilterError(format!(
240                    "cannot parse elem type from {filter_value}"
241                ))),
242            },
243            "ts_start" | "start_ts" => match parse_time_str(filter_value) {
244                Some(t) => Ok(Filter::TsStart(t.and_utc().timestamp() as f64)),
245                None => Err(FilterError(format!(
246                    "cannot parse TsStart filter from {filter_value}"
247                ))),
248            },
249            "ts_end" | "end_ts" => match parse_time_str(filter_value) {
250                Some(t) => Ok(Filter::TsEnd(t.and_utc().timestamp() as f64)),
251                None => Err(FilterError(format!(
252                    "cannot parse TsEnd filter from {filter_value}"
253                ))),
254            },
255            "as_path" => match ComparableRegex::new(filter_value) {
256                Ok(v) => Ok(Filter::AsPath(v)),
257                Err(_) => Err(FilterError(format!(
258                    "cannot parse AS path regex from {filter_value}"
259                ))),
260            },
261            "community" => match ComparableRegex::new(filter_value) {
262                Ok(v) => Ok(Filter::Community(v)),
263                Err(_) => Err(FilterError(format!(
264                    "cannot parse Community regex from {filter_value}"
265                ))),
266            },
267            "ip_version" | "ip" => match filter_value {
268                "4" | "v4" | "ipv4" => Ok(Filter::IpVersion(IpVersion::Ipv4)),
269                "6" | "v6" | "ipv6" => Ok(Filter::IpVersion(IpVersion::Ipv6)),
270                _ => Err(FilterError(format!(
271                    "cannot parse IP version from {filter_value}"
272                ))),
273            },
274            _ => Err(FilterError(format!("unknown filter type: {filter_type}"))),
275        }
276    }
277}
278
279pub trait Filterable {
280    fn match_filter(&self, filter: &Filter) -> bool;
281    fn match_filters(&self, filters: &[Filter]) -> bool;
282}
283
284const fn same_family(prefix_1: &IpNet, prefix_2: &IpNet) -> bool {
285    matches!(
286        (prefix_1, prefix_2),
287        (IpNet::V4(_), IpNet::V4(_)) | (IpNet::V6(_), IpNet::V6(_))
288    )
289}
290
291fn prefix_match(match_prefix: &IpNet, input_prefix: &IpNet, t: &PrefixMatchType) -> bool {
292    let exact = input_prefix.eq(match_prefix);
293    match t {
294        PrefixMatchType::Exact => exact,
295        PrefixMatchType::IncludeSuper => {
296            if exact {
297                exact
298            } else if !same_family(match_prefix, input_prefix) {
299                // version not match
300                false
301            } else {
302                // input_prefix is super prefix of match_prefix
303                match_prefix.addr() >= input_prefix.addr()
304                    && match_prefix.broadcast() <= input_prefix.broadcast()
305            }
306        }
307        PrefixMatchType::IncludeSub => {
308            if exact {
309                exact
310            } else if !same_family(match_prefix, input_prefix) {
311                // version not match
312                false
313            } else {
314                // input_prefix is sub prefix of match_prefix
315                match_prefix.addr() <= input_prefix.addr()
316                    && match_prefix.broadcast() >= input_prefix.broadcast()
317            }
318        }
319        PrefixMatchType::IncludeSuperSub => {
320            if exact {
321                exact
322            } else if !same_family(match_prefix, input_prefix) {
323                // version not match
324                false
325            } else {
326                // input_prefix is super prefix of match_prefix
327                (match_prefix.addr() >= input_prefix.addr()
328                    && match_prefix.broadcast() <= input_prefix.broadcast())
329                    || (match_prefix.addr() <= input_prefix.addr()
330                        && match_prefix.broadcast() >= input_prefix.broadcast())
331            }
332        }
333    }
334}
335
336impl Filterable for BgpElem {
337    fn match_filter(&self, filter: &Filter) -> bool {
338        match filter {
339            Filter::Negated(inner) => !self.match_filter(inner),
340            Filter::OriginAsn(v) => {
341                let asn: Asn = (*v).into();
342                if let Some(origins) = &self.origin_asns {
343                    origins.contains(&asn)
344                } else {
345                    false
346                }
347            }
348            Filter::Prefix(v, t) => prefix_match(v, &self.prefix.prefix, t),
349            Filter::PeerIp(v) => self.peer_ip == *v,
350            Filter::PeerIps(v) => v.contains(&self.peer_ip),
351            Filter::PeerAsn(v) => self.peer_asn.eq(v),
352            Filter::Type(v) => self.elem_type.eq(v),
353            Filter::TsStart(v) => self.timestamp >= *v,
354            Filter::TsEnd(v) => self.timestamp <= *v,
355            Filter::AsPath(v) => {
356                if let Some(path) = &self.as_path {
357                    v.is_match(path.to_string().as_str())
358                } else {
359                    false
360                }
361            }
362            Filter::Community(r) => {
363                if let Some(communities) = &self.communities {
364                    communities.iter().any(|c| r.is_match(c.to_string()))
365                } else {
366                    false
367                }
368            }
369            Filter::IpVersion(version) => match version {
370                IpVersion::Ipv4 => self.prefix.prefix.addr().is_ipv4(),
371                IpVersion::Ipv6 => self.prefix.prefix.addr().is_ipv6(),
372            },
373        }
374    }
375
376    fn match_filters(&self, filters: &[Filter]) -> bool {
377        filters.iter().all(|f| self.match_filter(f))
378    }
379}
380
381#[cfg(test)]
382mod tests {
383    use super::*;
384    use crate::BgpkitParser;
385    use anyhow::Result;
386    use std::str::FromStr;
387
388    #[test]
389    fn test_filters_on_mrt_file() {
390        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
391        let parser = BgpkitParser::new(url).unwrap();
392        let elems = parser.into_elem_iter().collect::<Vec<BgpElem>>();
393
394        let filters = vec![Filter::PeerIp(IpAddr::from_str("185.1.8.65").unwrap())];
395        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
396        assert_eq!(count, 3393);
397
398        let filters = vec![
399            Filter::PeerIp(IpAddr::from_str("185.1.8.65").unwrap()),
400            Filter::Prefix(
401                IpNet::from_str("190.115.192.0/22").unwrap(),
402                PrefixMatchType::Exact,
403            ),
404        ];
405        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
406        assert_eq!(count, 5);
407
408        let filters = vec![Filter::Prefix(
409            IpNet::from_str("190.115.192.0/24").unwrap(),
410            PrefixMatchType::IncludeSuper,
411        )];
412        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
413        assert_eq!(count, 18);
414
415        let filters = vec![Filter::Prefix(
416            IpNet::from_str("190.115.192.0/22").unwrap(),
417            PrefixMatchType::IncludeSub,
418        )];
419        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
420        assert_eq!(count, 42);
421
422        let filters = vec![Filter::Prefix(
423            IpNet::from_str("190.115.192.0/23").unwrap(),
424            PrefixMatchType::IncludeSuperSub,
425        )];
426        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
427        assert_eq!(count, 24);
428
429        let filters = vec![Filter::new("as_path", r" ?174 1916 52888$").unwrap()];
430        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
431        assert_eq!(count, 12);
432
433        // filter by community starting with some value
434        let filters = vec![Filter::new("community", r"60924:.*").unwrap()];
435        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
436        assert_eq!(count, 4243);
437
438        // filter by community ending with some value
439        let filters = vec![Filter::new("community", r".+:784$").unwrap()];
440        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
441        assert_eq!(count, 107);
442
443        // filter by community with large community (i.e. with 3 values, separated by ':')
444        let filters = vec![Filter::new("community", r"\d+:\d+:\d+$").unwrap()];
445        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
446        assert_eq!(count, 4397);
447
448        let filters = vec![
449            Filter::TsStart(1637437798_f64),
450            Filter::TsEnd(1637437798_f64),
451        ];
452        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
453        assert_eq!(count, 13);
454
455        let filters = vec![Filter::Type(ElemType::WITHDRAW)];
456        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
457        assert_eq!(count, 379);
458
459        let filters = vec![
460            Filter::Type(ElemType::WITHDRAW),
461            Filter::Prefix(
462                IpNet::from_str("2804:100::/32").unwrap(),
463                PrefixMatchType::Exact,
464            ),
465        ];
466        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
467        assert_eq!(count, 1);
468
469        // test filtering by multiple peers
470        /*
471        1167 185.1.8.3
472        1563 185.1.8.50
473        3393 185.1.8.65
474          51 185.1.8.89
475         834 2001:7f8:73:0:3:fa4:0:1
476          94 2001:7f8:73::c2a8:0:1
477        1058 2001:7f8:73::edfc:0:2
478         */
479        let filters = vec![Filter::PeerIps(vec![
480            IpAddr::from_str("185.1.8.65").unwrap(),
481            IpAddr::from_str("2001:7f8:73:0:3:fa4:0:1").unwrap(),
482        ])];
483        let count = elems.iter().filter(|e| e.match_filters(&filters)).count();
484        assert_eq!(count, 3393 + 834);
485    }
486
487    #[test]
488    fn test_filter_incorrect_filters() {
489        // filter by community with large community (i.e. with 3 values, separated by ':')
490        let incorrect_filters = [
491            Filter::new("community", r"[abc"),
492            Filter::new("as_path", r"[0-9"),
493            Filter::new("prefix_super_sub", "-192.-168.-1.1/24"),
494        ];
495        assert!(incorrect_filters
496            .iter()
497            .all(|f| matches!(f, Err(FilterError(_)))));
498    }
499
500    #[test]
501    fn test_parsing_time_str() {
502        let ts = chrono::NaiveDateTime::from_str("2021-11-20T19:49:58").unwrap();
503        assert_eq!(parse_time_str("1637437798"), Some(ts));
504        assert_eq!(parse_time_str("2021-11-20T19:49:58Z"), Some(ts));
505        assert_eq!(parse_time_str("2021-11-20T19:49:58+00:00"), Some(ts));
506
507        assert_eq!(parse_time_str("2021-11-20T19:49:58"), None);
508        assert_eq!(parse_time_str("2021-11-20T19:49:58ZDXV"), None);
509        assert_eq!(parse_time_str("2021-11-20 19:49:58"), None);
510        assert_eq!(parse_time_str("2021-11-20"), None);
511    }
512
513    #[test]
514    fn test_filter_iter() -> Result<()> {
515        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
516        let parser = BgpkitParser::new(url)?
517            .add_filter("peer_ip", "185.1.8.50")?
518            .add_filter("type", "w")?;
519        let count = parser.into_elem_iter().count();
520        assert_eq!(count, 39);
521
522        let parser = BgpkitParser::new(url)?
523            .add_filter("ts_start", "1637437798")?
524            .add_filter("ts_end", "2021-11-20T19:49:58Z")?;
525        let count = parser.into_elem_iter().count();
526        assert_eq!(count, 13);
527        Ok(())
528    }
529
530    #[test]
531    fn test_filter_iter_with_negation() -> Result<()> {
532        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
533
534        // Test negative filter with add_filter - exclude peer 185.1.8.65
535        // From test_filters_on_mrt_file, peer 185.1.8.65 has 3393 elements out of 8160 total
536        let parser = BgpkitParser::new(url)?.add_filter("!peer_ip", "185.1.8.65")?;
537        let count = parser.into_elem_iter().count();
538        assert_eq!(count, 8160 - 3393);
539
540        // Test negative type filter - get all non-withdrawals
541        // From test_filters_on_mrt_file, there are 379 withdrawals out of 8160 total
542        let parser = BgpkitParser::new(url)?.add_filter("!type", "w")?;
543        let count = parser.into_elem_iter().count();
544        assert_eq!(count, 8160 - 379);
545
546        // Test combining positive and negative filters
547        // Get elements from peer 185.1.8.50 that are NOT withdrawals
548        let parser = BgpkitParser::new(url)?
549            .add_filter("peer_ip", "185.1.8.50")?
550            .add_filter("!type", "w")?;
551        let count = parser.into_elem_iter().count();
552        // peer 185.1.8.50 has 1563 total, 39 withdrawals -> 1563 - 39 = 1524 non-withdrawals
553        assert_eq!(count, 1563 - 39);
554
555        Ok(())
556    }
557
558    #[test]
559    fn test_filter_iter_multi_peers() {
560        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
561        let parser = BgpkitParser::new(url)
562            .unwrap()
563            .add_filter("peer_ips", "185.1.8.65, 2001:7f8:73:0:3:fa4:0:1")
564            .unwrap();
565        let count = parser.into_elem_iter().count();
566        assert_eq!(count, 3393 + 834);
567    }
568
569    #[test]
570    fn test_prefix_match() {
571        // network
572        let p1 = IpNet::from_str("10.1.1.0/24").unwrap();
573        let p1_exact = IpNet::from_str("10.1.1.0/24").unwrap();
574        let p1_super = IpNet::from_str("10.1.0.0/16").unwrap();
575        let p1_sub = IpNet::from_str("10.1.1.0/25").unwrap();
576
577        let p2 = IpNet::from_str("2001:0DB8:0000:000b::/64").unwrap();
578
579        // exact
580        assert!(prefix_match(&p1, &p1_exact, &PrefixMatchType::Exact));
581        assert!(!prefix_match(&p1, &p1_sub, &PrefixMatchType::Exact));
582        assert!(!prefix_match(&p1, &p1_super, &PrefixMatchType::Exact));
583        assert!(!prefix_match(&p1, &p2, &PrefixMatchType::Exact));
584
585        // include super
586        assert!(prefix_match(&p1, &p1_exact, &PrefixMatchType::IncludeSuper));
587        assert!(!prefix_match(&p1, &p1_sub, &PrefixMatchType::IncludeSuper));
588        assert!(prefix_match(&p1, &p1_super, &PrefixMatchType::IncludeSuper));
589        assert!(!prefix_match(&p1, &p2, &PrefixMatchType::IncludeSuper));
590
591        // include sub
592        assert!(prefix_match(&p1, &p1_exact, &PrefixMatchType::IncludeSub));
593        assert!(prefix_match(&p1, &p1_sub, &PrefixMatchType::IncludeSub));
594        assert!(!prefix_match(&p1, &p1_super, &PrefixMatchType::IncludeSub));
595        assert!(!prefix_match(&p1, &p2, &PrefixMatchType::IncludeSub));
596
597        // include both
598        assert!(prefix_match(
599            &p1,
600            &p1_exact,
601            &PrefixMatchType::IncludeSuperSub
602        ));
603        assert!(prefix_match(
604            &p1,
605            &p1_sub,
606            &PrefixMatchType::IncludeSuperSub
607        ));
608        assert!(prefix_match(
609            &p1,
610            &p1_super,
611            &PrefixMatchType::IncludeSuperSub
612        ));
613        assert!(!prefix_match(&p1, &p2, &PrefixMatchType::IncludeSuperSub));
614    }
615
616    #[test]
617    fn test_filter_new() {
618        let filter = Filter::new("origin_asn", "12345").unwrap();
619        assert_eq!(filter, Filter::OriginAsn(12345));
620
621        // Test negated filters
622        let filter = Filter::new("!origin_asn", "12345").unwrap();
623        assert_eq!(filter, Filter::Negated(Box::new(Filter::OriginAsn(12345))));
624
625        let filter = Filter::new("!prefix", "192.168.1.0/24").unwrap();
626        assert_eq!(
627            filter,
628            Filter::Negated(Box::new(Filter::Prefix(
629                IpNet::from_str("192.168.1.0/24").unwrap(),
630                PrefixMatchType::Exact
631            )))
632        );
633
634        let filter = Filter::new("!peer_ip", "192.168.1.1").unwrap();
635        assert_eq!(
636            filter,
637            Filter::Negated(Box::new(Filter::PeerIp(
638                IpAddr::from_str("192.168.1.1").unwrap()
639            )))
640        );
641
642        let filter = Filter::new("!peer_asn", "12345").unwrap();
643        assert_eq!(filter, Filter::Negated(Box::new(Filter::PeerAsn(12345))));
644
645        let filter = Filter::new("!type", "w").unwrap();
646        assert_eq!(
647            filter,
648            Filter::Negated(Box::new(Filter::Type(ElemType::WITHDRAW)))
649        );
650
651        let filter = Filter::new("!ip_version", "4").unwrap();
652        assert_eq!(
653            filter,
654            Filter::Negated(Box::new(Filter::IpVersion(IpVersion::Ipv4)))
655        );
656
657        let filter = Filter::new("prefix", "192.168.1.0/24").unwrap();
658        assert_eq!(
659            filter,
660            Filter::Prefix(
661                IpNet::from_str("192.168.1.0/24").unwrap(),
662                PrefixMatchType::Exact
663            )
664        );
665        let filter = Filter::new("prefix_super", "192.168.1.0/24").unwrap();
666        assert_eq!(
667            filter,
668            Filter::Prefix(
669                IpNet::from_str("192.168.1.0/24").unwrap(),
670                PrefixMatchType::IncludeSuper
671            )
672        );
673        let filter = Filter::new("prefix_sub", "192.168.1.0/24").unwrap();
674        assert_eq!(
675            filter,
676            Filter::Prefix(
677                IpNet::from_str("192.168.1.0/24").unwrap(),
678                PrefixMatchType::IncludeSub
679            )
680        );
681        let filter = Filter::new("prefix_super_sub", "192.168.1.0/24").unwrap();
682        assert_eq!(
683            filter,
684            Filter::Prefix(
685                IpNet::from_str("192.168.1.0/24").unwrap(),
686                PrefixMatchType::IncludeSuperSub
687            )
688        );
689
690        let filter = Filter::new("peer_ip", "192.168.1.1").unwrap();
691        assert_eq!(
692            filter,
693            Filter::PeerIp(IpAddr::from_str("192.168.1.1").unwrap())
694        );
695
696        let filter = Filter::new("peer_asn", "12345").unwrap();
697        assert_eq!(filter, Filter::PeerAsn(12345));
698
699        let filter = Filter::new("type", "w").unwrap();
700        assert_eq!(filter, Filter::Type(ElemType::WITHDRAW));
701
702        let filter = Filter::new("ts_start", "1637437798").unwrap();
703        assert_eq!(filter, Filter::TsStart(1637437798_f64));
704
705        let filter = Filter::new("ts_end", "1637437798").unwrap();
706        assert_eq!(filter, Filter::TsEnd(1637437798_f64));
707
708        let filter = Filter::new("as_path", r" ?174 1916 52888$").unwrap();
709        assert_eq!(
710            filter,
711            Filter::AsPath(ComparableRegex::new(r" ?174 1916 52888$").unwrap())
712        );
713
714        assert!(Filter::new("origin_asn", "not a number").is_err());
715        assert!(Filter::new("peer_asn", "not a number").is_err());
716        assert!(Filter::new("ts_start", "not a number").is_err());
717        assert!(Filter::new("ts_end", "not a number").is_err());
718        assert!(Filter::new("prefix", "not a prefix").is_err());
719        assert!(Filter::new("prefix_super", "not a prefix").is_err());
720        assert!(Filter::new("prefix_sub", "not a prefix").is_err());
721        assert!(Filter::new("peer_ip", "not a IP").is_err());
722        assert!(Filter::new("peer_ips", "not,a,IP").is_err());
723        assert!(Filter::new("type", "not a type").is_err());
724        assert!(Filter::new("as_path", "[abc").is_err());
725        assert!(Filter::new("ip_version", "5").is_err());
726        assert!(Filter::new("unknown_filter", "some_value").is_err());
727    }
728
729    #[test]
730    fn test_filterable_match_filter() {
731        let elem = BgpElem {
732            timestamp: 1637437798_f64,
733            peer_ip: IpAddr::from_str("192.168.1.1").unwrap(),
734            peer_asn: Asn::new_32bit(12345),
735            prefix: NetworkPrefix::new(IpNet::from_str("192.168.1.0/24").unwrap(), None),
736            next_hop: None,
737            as_path: Some(AsPath::from_sequence(vec![174, 1916, 52888])),
738            origin_asns: Some(vec![Asn::new_16bit(12345)]),
739            origin: None,
740            local_pref: None,
741            med: None,
742            communities: Some(vec![MetaCommunity::Large(LargeCommunity::new(
743                12345,
744                [678910, 111213],
745            ))]),
746            atomic: false,
747            aggr_asn: None,
748            aggr_ip: None,
749            only_to_customer: None,
750            unknown: None,
751            elem_type: ElemType::ANNOUNCE,
752            deprecated: None,
753        };
754
755        let mut filters = vec![];
756
757        let filter = Filter::new("origin_asn", "12345").unwrap();
758        filters.push(filter.clone());
759        assert!(elem.match_filter(&filter));
760
761        let filter = Filter::new("origin_asn", "678910").unwrap();
762        assert!(!elem.match_filter(&filter));
763
764        let filter = Filter::new("prefix", "192.168.1.0/24").unwrap();
765        filters.push(filter.clone());
766        assert!(elem.match_filter(&filter));
767
768        let filter = Filter::new("peer_ip", "192.168.1.1").unwrap();
769        filters.push(filter.clone());
770        assert!(elem.match_filter(&filter));
771
772        let filter = Filter::new("peer_asn", "12345").unwrap();
773        filters.push(filter.clone());
774        assert!(elem.match_filter(&filter));
775
776        let filter = Filter::new("type", "a").unwrap();
777        filters.push(filter.clone());
778        assert!(elem.match_filter(&filter));
779
780        let filter = Filter::new("ts_start", "1637437798").unwrap();
781        filters.push(filter.clone());
782        assert!(elem.match_filter(&filter));
783
784        let filter = Filter::new("ts_end", "1637437798").unwrap();
785        filters.push(filter.clone());
786        assert!(elem.match_filter(&filter));
787
788        let filter = Filter::new("as_path", r" ?174 1916 52888$").unwrap();
789        filters.push(filter.clone());
790        assert!(elem.match_filter(&filter));
791
792        let filter = Filter::new("ip_version", "4").unwrap();
793        filters.push(filter.clone());
794        assert!(elem.match_filter(&filter));
795
796        let filter = Filter::new("ip", "ipv6").unwrap();
797        assert!(!elem.match_filter(&filter));
798
799        let filter = Filter::new("community", r"12345:678910:111213$").unwrap();
800        filters.push(filter.clone());
801        assert!(elem.match_filter(&filter));
802
803        assert!(elem.match_filters(&filters));
804    }
805
806    #[test]
807    fn test_negated_filters() {
808        let elem = BgpElem {
809            timestamp: 1637437798_f64,
810            peer_ip: IpAddr::from_str("192.168.1.1").unwrap(),
811            peer_asn: Asn::new_32bit(12345),
812            prefix: NetworkPrefix::new(IpNet::from_str("192.168.1.0/24").unwrap(), None),
813            next_hop: None,
814            as_path: Some(AsPath::from_sequence(vec![174, 1916, 52888])),
815            origin_asns: Some(vec![Asn::new_16bit(12345)]),
816            origin: None,
817            local_pref: None,
818            med: None,
819            communities: Some(vec![MetaCommunity::Large(LargeCommunity::new(
820                12345,
821                [678910, 111213],
822            ))]),
823            atomic: false,
824            aggr_asn: None,
825            aggr_ip: None,
826            only_to_customer: None,
827            unknown: None,
828            elem_type: ElemType::ANNOUNCE,
829            deprecated: None,
830        };
831
832        // Test negated origin_asn filter
833        // elem has origin_asn 12345, so !origin_asn=12345 should NOT match
834        let filter = Filter::new("!origin_asn", "12345").unwrap();
835        assert!(!elem.match_filter(&filter));
836
837        // elem has origin_asn 12345, so !origin_asn=99999 should match
838        let filter = Filter::new("!origin_asn", "99999").unwrap();
839        assert!(elem.match_filter(&filter));
840
841        // Test negated prefix filter
842        // elem has prefix 192.168.1.0/24, so !prefix=192.168.1.0/24 should NOT match
843        let filter = Filter::new("!prefix", "192.168.1.0/24").unwrap();
844        assert!(!elem.match_filter(&filter));
845
846        // elem has prefix 192.168.1.0/24, so !prefix=10.0.0.0/8 should match
847        let filter = Filter::new("!prefix", "10.0.0.0/8").unwrap();
848        assert!(elem.match_filter(&filter));
849
850        // Test negated peer_ip filter
851        // elem has peer_ip 192.168.1.1, so !peer_ip=192.168.1.1 should NOT match
852        let filter = Filter::new("!peer_ip", "192.168.1.1").unwrap();
853        assert!(!elem.match_filter(&filter));
854
855        // elem has peer_ip 192.168.1.1, so !peer_ip=10.0.0.1 should match
856        let filter = Filter::new("!peer_ip", "10.0.0.1").unwrap();
857        assert!(elem.match_filter(&filter));
858
859        // Test negated peer_asn filter
860        // elem has peer_asn 12345, so !peer_asn=12345 should NOT match
861        let filter = Filter::new("!peer_asn", "12345").unwrap();
862        assert!(!elem.match_filter(&filter));
863
864        // elem has peer_asn 12345, so !peer_asn=99999 should match
865        let filter = Filter::new("!peer_asn", "99999").unwrap();
866        assert!(elem.match_filter(&filter));
867
868        // Test negated type filter
869        // elem has type ANNOUNCE, so !type=a should NOT match
870        let filter = Filter::new("!type", "a").unwrap();
871        assert!(!elem.match_filter(&filter));
872
873        // elem has type ANNOUNCE, so !type=w should match
874        let filter = Filter::new("!type", "w").unwrap();
875        assert!(elem.match_filter(&filter));
876
877        // Test negated ip_version filter
878        // elem has IPv4 prefix, so !ip_version=4 should NOT match
879        let filter = Filter::new("!ip_version", "4").unwrap();
880        assert!(!elem.match_filter(&filter));
881
882        // elem has IPv4 prefix, so !ip_version=6 should match
883        let filter = Filter::new("!ip_version", "6").unwrap();
884        assert!(elem.match_filter(&filter));
885
886        // Test negated as_path filter
887        // elem has as_path "174 1916 52888", so negated matching regex should NOT match
888        let filter = Filter::new("!as_path", r"174 1916 52888$").unwrap();
889        assert!(!elem.match_filter(&filter));
890
891        // elem has as_path "174 1916 52888", so negated non-matching regex should match
892        let filter = Filter::new("!as_path", r"99999$").unwrap();
893        assert!(elem.match_filter(&filter));
894
895        // Test negated community filter
896        let filter = Filter::new("!community", r"12345:678910:111213$").unwrap();
897        assert!(!elem.match_filter(&filter));
898
899        let filter = Filter::new("!community", r"99999:99999$").unwrap();
900        assert!(elem.match_filter(&filter));
901
902        // Test negated peer_ips filter
903        let filter = Filter::new("!peer_ips", "192.168.1.1, 10.0.0.1").unwrap();
904        assert!(!elem.match_filter(&filter)); // elem's peer_ip is in the list
905
906        let filter = Filter::new("!peer_ips", "10.0.0.1, 10.0.0.2").unwrap();
907        assert!(elem.match_filter(&filter)); // elem's peer_ip is NOT in the list
908
909        // Test combining positive and negated filters
910        let filters = vec![
911            Filter::new("origin_asn", "12345").unwrap(),   // matches
912            Filter::new("!peer_asn", "99999").unwrap(),    // matches (not 99999)
913            Filter::new("!prefix", "10.0.0.0/8").unwrap(), // matches (not 10.0.0.0/8)
914        ];
915        assert!(elem.match_filters(&filters));
916
917        // Test combining filters where one fails
918        let filters = vec![
919            Filter::new("origin_asn", "12345").unwrap(),  // matches
920            Filter::new("!origin_asn", "12345").unwrap(), // does NOT match
921        ];
922        assert!(!elem.match_filters(&filters));
923    }
924
925    #[test]
926    fn test_negated_filters_on_mrt_file() {
927        let url = "https://spaces.bgpkit.org/parser/update-example.gz";
928        let parser = BgpkitParser::new(url).unwrap();
929        let elems = parser.into_elem_iter().collect::<Vec<BgpElem>>();
930
931        // Count all elems from peer 185.1.8.65
932        let filters = vec![Filter::PeerIp(IpAddr::from_str("185.1.8.65").unwrap())];
933        let count_with_peer = elems.iter().filter(|e| e.match_filters(&filters)).count();
934        assert_eq!(count_with_peer, 3393);
935
936        // Count all elems NOT from peer 185.1.8.65
937        let filters = vec![Filter::new("!peer_ip", "185.1.8.65").unwrap()];
938        let count_without_peer = elems.iter().filter(|e| e.match_filters(&filters)).count();
939        assert_eq!(count_without_peer, elems.len() - 3393);
940
941        // Verify total adds up
942        assert_eq!(count_with_peer + count_without_peer, elems.len());
943
944        // Test negated type filter
945        let filters = vec![Filter::Type(ElemType::WITHDRAW)];
946        let count_withdrawals = elems.iter().filter(|e| e.match_filters(&filters)).count();
947        assert_eq!(count_withdrawals, 379);
948
949        let filters = vec![Filter::new("!type", "w").unwrap()];
950        let count_not_withdrawals = elems.iter().filter(|e| e.match_filters(&filters)).count();
951        assert_eq!(count_not_withdrawals, elems.len() - 379);
952
953        // Test negated prefix filter
954        let filters = vec![Filter::Prefix(
955            IpNet::from_str("190.115.192.0/22").unwrap(),
956            PrefixMatchType::Exact,
957        )];
958        let count_with_prefix = elems.iter().filter(|e| e.match_filters(&filters)).count();
959
960        let filters = vec![Filter::new("!prefix", "190.115.192.0/22").unwrap()];
961        let count_without_prefix = elems.iter().filter(|e| e.match_filters(&filters)).count();
962        assert_eq!(count_with_prefix + count_without_prefix, elems.len());
963
964        // Test negated prefix_super filter
965        let filters = vec![Filter::Prefix(
966            IpNet::from_str("190.115.192.0/24").unwrap(),
967            PrefixMatchType::IncludeSuper,
968        )];
969        let count_with_super = elems.iter().filter(|e| e.match_filters(&filters)).count();
970
971        let filters = vec![Filter::new("!prefix_super", "190.115.192.0/24").unwrap()];
972        let count_without_super = elems.iter().filter(|e| e.match_filters(&filters)).count();
973        assert_eq!(count_with_super + count_without_super, elems.len());
974
975        // Test negated prefix_sub filter
976        let filters = vec![Filter::Prefix(
977            IpNet::from_str("190.115.192.0/22").unwrap(),
978            PrefixMatchType::IncludeSub,
979        )];
980        let count_with_sub = elems.iter().filter(|e| e.match_filters(&filters)).count();
981
982        let filters = vec![Filter::new("!prefix_sub", "190.115.192.0/22").unwrap()];
983        let count_without_sub = elems.iter().filter(|e| e.match_filters(&filters)).count();
984        assert_eq!(count_with_sub + count_without_sub, elems.len());
985
986        // Test negated prefix_super_sub filter
987        let filters = vec![Filter::Prefix(
988            IpNet::from_str("190.115.192.0/23").unwrap(),
989            PrefixMatchType::IncludeSuperSub,
990        )];
991        let count_with_super_sub = elems.iter().filter(|e| e.match_filters(&filters)).count();
992
993        let filters = vec![Filter::new("!prefix_super_sub", "190.115.192.0/23").unwrap()];
994        let count_without_super_sub = elems.iter().filter(|e| e.match_filters(&filters)).count();
995        assert_eq!(count_with_super_sub + count_without_super_sub, elems.len());
996    }
997
998    #[test]
999    fn test_double_negation_rejected() {
1000        // Double negation should be rejected with a clear error message
1001        let result = Filter::new("!!origin_asn", "13335");
1002        assert!(result.is_err());
1003        let err = result.unwrap_err();
1004        assert!(err.to_string().contains("double negation"));
1005
1006        let result = Filter::new("!!!prefix", "10.0.0.0/8");
1007        assert!(result.is_err());
1008        let err = result.unwrap_err();
1009        assert!(err.to_string().contains("double negation"));
1010    }
1011
1012    #[test]
1013    fn test_timestamp_negation_rejected() {
1014        // Timestamp filter negation should be rejected
1015        let result = Filter::new("!ts_start", "1637437798");
1016        assert!(result.is_err());
1017        let err = result.unwrap_err();
1018        assert!(err
1019            .to_string()
1020            .contains("timestamp filters do not support negation"));
1021
1022        let result = Filter::new("!ts_end", "1637437798");
1023        assert!(result.is_err());
1024        let err = result.unwrap_err();
1025        assert!(err
1026            .to_string()
1027            .contains("timestamp filters do not support negation"));
1028
1029        let result = Filter::new("!start_ts", "1637437798");
1030        assert!(result.is_err());
1031
1032        let result = Filter::new("!end_ts", "1637437798");
1033        assert!(result.is_err());
1034    }
1035}
bgpkit_parser/parser/filter.rs

bgpkit_parser/parser/
filter.rs
