Skip to main content

bgpkit_broker/
lib.rs

1/*!
2# Overview
3
4[bgpkit-broker][crate] is a package that allows accessing the BGPKIT Broker API and search for BGP archive
5files with different search parameters available.
6
7# Examples
8
9## Basic Usage with Iterator
10
11The recommended usage to collect [BrokerItem]s is to use the built-in iterator. The
12[BrokerItemIterator] handles making API queries so that it can continuously stream new items until
13it reaches the end of items. This is useful for simply getting **all** matching items without need
14to worry about pagination.
15
16```no_run
17use bgpkit_broker::{BgpkitBroker, BrokerItem};
18
19let broker = BgpkitBroker::new()
20    .ts_start("2022-01-01")
21    .ts_end("2022-01-02")
22    .collector_id("route-views2");
23
24// Iterate by reference (reusable broker)
25for item in &broker {
26    println!("BGP file: {} from {} ({})",
27             item.url, item.collector_id, item.data_type);
28}
29
30// Or collect into vector
31let items: Vec<BrokerItem> = broker.into_iter().collect();
32println!("Found {} BGP archive files", items.len());
33```
34
35## Practical BGP Data Analysis with Shortcuts
36
37The SDK provides convenient shortcuts for common BGP data analysis patterns:
38
39### Daily RIB Analysis Across Diverse Collectors
40
41```no_run
42use bgpkit_broker::BgpkitBroker;
43
44// Find the most diverse collectors for comprehensive analysis
45let broker = BgpkitBroker::new()
46    .ts_start("2024-01-01")
47    .ts_end("2024-01-31");
48
49let diverse_collectors = broker.most_diverse_collectors(5, None).unwrap();
50println!("Selected {} diverse collectors: {:?}",
51         diverse_collectors.len(), diverse_collectors);
52
53// Get daily RIB snapshots from these collectors
54let daily_ribs = broker
55    .clone()
56    .collector_id(&diverse_collectors.join(","))
57    .daily_ribs().unwrap();
58
59println!("Found {} daily RIB snapshots for analysis", daily_ribs.len());
60for rib in daily_ribs.iter().take(3) {
61    println!("Daily snapshot: {} from {} at {}",
62             rib.collector_id,
63             rib.ts_start.format("%Y-%m-%d"),
64             rib.url);
65}
66```
67
68### Recent BGP Updates Monitoring
69
70```no_run
71use bgpkit_broker::BgpkitBroker;
72
73// Monitor recent BGP updates from multiple collectors
74let recent_updates = BgpkitBroker::new()
75    .collector_id("route-views2,rrc00,route-views6")
76    .recent_updates(6).unwrap(); // last 6 hours
77
78println!("Found {} recent BGP update files", recent_updates.len());
79for update in recent_updates.iter().take(5) {
80    println!("Update: {} from {} at {}",
81             update.collector_id,
82             update.ts_start.format("%Y-%m-%d %H:%M:%S"),
83             update.url);
84}
85```
86
87### Project-specific Analysis
88
89```no_run
90use bgpkit_broker::BgpkitBroker;
91
92// Compare RouteViews vs RIPE RIS daily snapshots
93let routeviews_ribs = BgpkitBroker::new()
94    .ts_start("2024-01-01")
95    .ts_end("2024-01-07")
96    .project("routeviews")
97    .daily_ribs().unwrap();
98
99let ripe_ribs = BgpkitBroker::new()
100    .ts_start("2024-01-01")
101    .ts_end("2024-01-07")
102    .project("riperis")
103    .daily_ribs().unwrap();
104
105println!("RouteViews daily RIBs: {}", routeviews_ribs.len());
106println!("RIPE RIS daily RIBs: {}", ripe_ribs.len());
107```
108
109### Advanced Collector Selection
110
111```no_run
112use bgpkit_broker::BgpkitBroker;
113
114let broker = BgpkitBroker::new();
115
116// Get diverse RouteViews collectors for focused analysis
117let rv_collectors = broker.most_diverse_collectors(3, Some("routeviews")).unwrap();
118println!("Diverse RouteViews collectors: {:?}", rv_collectors);
119
120// Use them to get comprehensive recent updates
121let comprehensive_updates = broker
122    .clone()
123    .collector_id(&rv_collectors.join(","))
124    .recent_updates(12).unwrap(); // last 12 hours
125
126println!("Got {} updates from {} collectors",
127         comprehensive_updates.len(), rv_collectors.len());
128```
129
130### Routing Table Snapshot Reconstruction
131
132```no_run
133use bgpkit_broker::BgpkitBroker;
134
135// Get the MRT files needed to construct a routing table snapshot
136let broker = BgpkitBroker::new();
137let snapshots = broker.get_snapshot_files(
138    &["route-views2", "rrc00"],
139    "2024-01-01T12:00:00Z"
140).unwrap();
141
142for snapshot in snapshots {
143    println!("Collector: {}", snapshot.collector_id);
144    println!("RIB dump: {}", snapshot.rib_url);
145    println!("Updates to apply: {}", snapshot.updates_urls.len());
146
147    // Use with bgpkit-parser to reconstruct routing table:
148    // 1. Parse RIB dump for initial state
149    // 2. Apply updates in order to reach target timestamp
150}
151```
152
153## Manual Page Queries
154
155For fine-grained control over pagination or custom iteration patterns:
156
157```rust,no_run
158use bgpkit_broker::BgpkitBroker;
159
160let mut broker = BgpkitBroker::new()
161    .ts_start("2022-01-01")
162    .ts_end("2022-01-02")
163    .page(1)
164    .page_size(50);
165
166// Query specific page
167let page1_items = broker.query_single_page().unwrap();
168println!("Page 1: {} items", page1_items.len());
169
170// Move to next page
171broker.turn_page(2);
172let page2_items = broker.query_single_page().unwrap();
173println!("Page 2: {} items", page2_items.len());
174```
175
176## Getting Latest Files and Peer Information
177
178Access the most recent data and peer information:
179
180```rust,no_run
181use bgpkit_broker::BgpkitBroker;
182
183// Get latest files from all collectors
184let broker = BgpkitBroker::new();
185let latest_files = broker.latest().unwrap();
186println!("Latest files from {} collectors", latest_files.len());
187
188// Get full-feed peers from specific collector
189let peers = BgpkitBroker::new()
190    .collector_id("route-views2")
191    .peers_only_full_feed(true)
192    .get_peers().unwrap();
193
194println!("Found {} full-feed peers", peers.len());
195for peer in peers.iter().take(3) {
196    println!("Peer: AS{} ({}) - v4: {}, v6: {}",
197             peer.asn, peer.ip, peer.num_v4_pfxs, peer.num_v6_pfxs);
198}
199```
200*/
201
202#![doc(
203    html_logo_url = "https://raw.githubusercontent.com/bgpkit/assets/main/logos/icon-transparent.png",
204    html_favicon_url = "https://raw.githubusercontent.com/bgpkit/assets/main/logos/favicon.ico"
205)]
206#![allow(unknown_lints)]
207
208mod collector;
209#[cfg(feature = "cli")]
210pub mod config;
211#[cfg(feature = "cli")]
212pub mod crawler;
213#[cfg(feature = "backend")]
214pub mod db;
215mod error;
216mod item;
217#[cfg(feature = "nats")]
218pub mod notifier;
219mod peer;
220mod query;
221mod shortcuts;
222#[cfg(feature = "sse")]
223mod sse;
224
225use crate::collector::DEFAULT_COLLECTORS_CONFIG;
226use crate::peer::BrokerPeersResult;
227use crate::query::{BrokerQueryResult, CollectorLatestResult};
228use chrono::{DateTime, NaiveDate, TimeZone, Utc};
229pub use collector::{load_collectors, Collector};
230
231#[cfg(feature = "cli")]
232pub use config::{BrokerConfig, DatabaseTarget};
233#[cfg(feature = "cli")]
234pub use crawler::crawl_collector;
235#[cfg(feature = "cli")]
236pub use db::DatabaseBackend;
237#[cfg(feature = "backend")]
238pub use db::{LocalBrokerDb, PostgresDb, UpdatesMeta, DEFAULT_PAGE_SIZE};
239pub use error::BrokerError;
240pub use item::BrokerItem;
241pub use peer::BrokerPeer;
242pub use query::{QueryParams, SortOrder};
243pub use shortcuts::SnapshotFiles;
244#[cfg(feature = "sse")]
245pub use sse::{BrokerItemSubscription, SseSubscriptionOptions};
246use std::collections::{HashMap, HashSet};
247use std::fmt::Display;
248use std::net::IpAddr;
249use std::path::PathBuf;
250
251const SDK_USER_AGENT: &str = concat!("bgpkit-broker/", env!("CARGO_PKG_VERSION"));
252
253/// BgpkitBroker struct maintains the broker's URL and handles making API queries.
254///
255/// See [module doc][crate#examples] for usage examples.
256#[derive(Clone)]
257pub struct BgpkitBroker {
258    pub broker_url: String,
259    pub query_params: QueryParams,
260    client: reqwest::blocking::Client,
261    collector_project_map: HashMap<String, String>,
262    accept_invalid_certs: bool,
263    cache_dir: Option<PathBuf>,
264}
265
266/// Default public broker API endpoint.
267const DEFAULT_BROKER_URL: &str = "https://api.bgpkit.com/v3/broker";
268
269/// Resolve the effective broker base URL: the `BGPKIT_BROKER_URL` value when
270/// set and non-blank, otherwise the default public API endpoint.
271fn resolve_broker_url() -> String {
272    normalize_broker_url(std::env::var("BGPKIT_BROKER_URL").ok())
273}
274
275fn normalize_broker_url(value: Option<String>) -> String {
276    match value {
277        Some(url) => {
278            let trimmed = url.trim();
279            if trimmed.is_empty() {
280                DEFAULT_BROKER_URL.to_string()
281            } else {
282                trimmed.trim_end_matches('/').to_string()
283            }
284        }
285        None => DEFAULT_BROKER_URL.to_string(),
286    }
287}
288
289impl Default for BgpkitBroker {
290    fn default() -> Self {
291        dotenvy::dotenv().ok();
292        let url = resolve_broker_url();
293
294        let collector_project_map = DEFAULT_COLLECTORS_CONFIG.clone().to_project_map();
295
296        let accept_invalid_certs = read_accept_invalid_certs_from_env();
297        let client = build_blocking_client(accept_invalid_certs);
298
299        Self {
300            broker_url: url,
301            query_params: Default::default(),
302            client,
303            collector_project_map,
304            accept_invalid_certs,
305            cache_dir: None,
306        }
307    }
308}
309
310fn read_accept_invalid_certs_from_env() -> bool {
311    match std::env::var("ONEIO_ACCEPT_INVALID_CERTS") {
312        Ok(t) => {
313            let l = t.to_lowercase();
314            l.starts_with("true") || l.starts_with("y")
315        }
316        Err(_) => false,
317    }
318}
319
320fn build_blocking_client(accept_invalid_certs: bool) -> reqwest::blocking::Client {
321    match reqwest::blocking::ClientBuilder::new()
322        .danger_accept_invalid_certs(accept_invalid_certs)
323        .user_agent(SDK_USER_AGENT)
324        .build()
325    {
326        Ok(c) => c,
327        Err(e) => {
328            panic!("Failed to build HTTP client for broker requests: {}", e);
329        }
330    }
331}
332
333#[cfg(feature = "sse")]
334pub(crate) fn build_async_client(
335    accept_invalid_certs: bool,
336) -> Result<reqwest::Client, BrokerError> {
337    reqwest::ClientBuilder::new()
338        .danger_accept_invalid_certs(accept_invalid_certs)
339        .user_agent(SDK_USER_AGENT)
340        .build()
341        .map_err(BrokerError::NetworkError)
342}
343
344impl BgpkitBroker {
345    /// Construct a new BgpkitBroker object.
346    ///
347    /// The URL and query parameters can be adjusted with other functions.
348    ///
349    /// Users can opt in to accept invalid SSL certificates by setting the environment variable
350    /// `ONEIO_ACCEPT_INVALID_CERTS` to `true`.
351    ///
352    /// # Examples
353    /// ```
354    /// use bgpkit_broker::BgpkitBroker;
355    /// let broker = BgpkitBroker::new();
356    /// ```
357    pub fn new() -> Self {
358        Self::default()
359    }
360
361    /// Configure broker URL.
362    ///
363    /// You can change the default broker URL to point to your own broker instance.
364    /// You can also change the URL by setting the environment variable `BGPKIT_BROKER_URL`.
365    ///
366    /// # Examples
367    /// ```
368    /// let broker = bgpkit_broker::BgpkitBroker::new()
369    ///     .broker_url("api.broker.example.com/v3");
370    /// ```
371    pub fn broker_url<S: Display>(self, url: S) -> Self {
372        let broker_url = url.to_string().trim_end_matches('/').to_string();
373        Self {
374            broker_url,
375            query_params: self.query_params,
376            client: self.client,
377            collector_project_map: self.collector_project_map,
378            accept_invalid_certs: self.accept_invalid_certs,
379            cache_dir: self.cache_dir,
380        }
381    }
382
383    /// DANGER: Accept invalid SSL certificates.
384    pub fn accept_invalid_certs(self) -> Self {
385        Self {
386            broker_url: self.broker_url,
387            query_params: self.query_params,
388            client: build_blocking_client(true),
389            collector_project_map: self.collector_project_map,
390            accept_invalid_certs: true,
391            cache_dir: self.cache_dir,
392        }
393    }
394
395    /// Disable SSL certificate check.
396    #[deprecated(since = "0.7.1", note = "Please use `accept_invalid_certs` instead.")]
397    pub fn disable_ssl_check(self) -> Self {
398        Self::accept_invalid_certs(self)
399    }
400
401    /// Set the cache directory for storing query results.
402    ///
403    /// When a cache directory is specified, query results will be cached to disk
404    /// and loaded from cache on subsequent queries with the same parameters.
405    /// This is useful for development and offline usage.
406    ///
407    /// The directory will be created if it doesn't exist. Panics if unable to create.
408    ///
409    /// # Examples
410    ///
411    /// ```no_run
412    /// let broker = bgpkit_broker::BgpkitBroker::new()
413    ///     .cache_dir("/tmp/bgpkit-cache");
414    /// ```
415    pub fn cache_dir<P: Into<PathBuf>>(mut self, path: P) -> Self {
416        let path = path.into();
417        if !path.exists() {
418            std::fs::create_dir_all(&path).expect("Failed to create cache directory");
419        }
420        self.cache_dir = Some(path);
421        self
422    }
423
424    /// Generate cache key from current query parameters.
425    fn cache_key(&self) -> String {
426        use sha2::{Digest, Sha256};
427
428        let params_str = format!(
429            "{}:{}:{}:{}:{}:{}:{}:{}",
430            self.broker_url,
431            self.query_params.ts_start.as_deref().unwrap_or(""),
432            self.query_params.ts_end.as_deref().unwrap_or(""),
433            self.query_params.collector_id.as_deref().unwrap_or(""),
434            self.query_params.project.as_deref().unwrap_or(""),
435            self.query_params.data_type.as_deref().unwrap_or(""),
436            self.query_params.page,
437            self.query_params.page_size
438        );
439
440        let mut hasher = Sha256::new();
441        hasher.update(params_str.as_bytes());
442        hasher
443            .finalize()
444            .iter()
445            .map(|b| format!("{:02x}", b))
446            .collect::<String>()
447    }
448
449    /// Try to load cached results for current query parameters.
450    fn load_cache(&self) -> Option<Vec<BrokerItem>> {
451        let cache_dir = self.cache_dir.as_ref()?;
452        let cache_file = cache_dir.join(self.cache_key()).with_extension("json");
453
454        if !cache_file.exists() {
455            return None;
456        }
457
458        match std::fs::read_to_string(&cache_file) {
459            Ok(contents) => match serde_json::from_str::<Vec<BrokerItem>>(&contents) {
460                Ok(items) => {
461                    log::info!("Loaded {} items from cache", items.len());
462                    Some(items)
463                }
464                Err(e) => {
465                    log::warn!("Failed to deserialize cache file: {}", e);
466                    None
467                }
468            },
469            Err(e) => {
470                log::warn!("Failed to read cache file: {}", e);
471                None
472            }
473        }
474    }
475
476    /// Save results to cache for current query parameters.
477    fn save_cache(&self, items: &[BrokerItem]) {
478        let Some(cache_dir) = self.cache_dir.as_ref() else {
479            return;
480        };
481
482        let cache_file = cache_dir.join(self.cache_key()).with_extension("json");
483
484        match serde_json::to_string(items) {
485            Ok(json) => {
486                if let Err(e) = std::fs::write(&cache_file, json) {
487                    log::warn!("Failed to write cache file: {}", e);
488                } else {
489                    log::info!("Saved {} items to cache", items.len());
490                }
491            }
492            Err(e) => {
493                log::warn!("Failed to serialize items for cache: {}", e);
494            }
495        }
496    }
497
498    /// Parse and validate timestamp string with support for multiple formats.
499    ///
500    /// Supported formats:
501    /// - Unix timestamp: "1640995200"
502    /// - RFC3339/ISO8601: "2022-01-01T00:00:00Z", "2022-01-01T12:30:45Z"
503    /// - RFC3339 without Z: "2022-01-01T00:00:00", "2022-01-01T12:30:45"
504    /// - Date with time: "2022-01-01 00:00:00", "2022-01-01 12:30:45"
505    /// - Pure date (start of day): "2022-01-01", "2022/01/01"
506    /// - Pure date with dots: "2022.01.01"
507    /// - Compact date: "20220101"
508    ///
509    /// For pure date formats, the time component defaults to 00:00:00 (start of day).
510    /// Returns a `DateTime<Utc>` for consistent handling and formatting.
511    fn parse_timestamp(timestamp: &str) -> Result<DateTime<Utc>, BrokerError> {
512        let ts_str = timestamp.trim();
513
514        // Try parsing as RFC3339 with timezone (including +00:00, -05:00, Z, etc.)
515        if let Ok(dt_with_tz) = DateTime::parse_from_rfc3339(ts_str) {
516            return Ok(dt_with_tz.with_timezone(&Utc));
517        }
518
519        // Try parsing as RFC3339/ISO8601 with Z
520        if let Ok(naive_dt) = chrono::NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%dT%H:%M:%SZ") {
521            return Ok(Utc.from_utc_datetime(&naive_dt));
522        }
523
524        // Try parsing as RFC3339 without Z (assume UTC)
525        if let Ok(naive_dt) = chrono::NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%dT%H:%M:%S") {
526            return Ok(Utc.from_utc_datetime(&naive_dt));
527        }
528
529        // Try parsing as "YYYY-MM-DD HH:MM:SS" (assume UTC)
530        if let Ok(naive_dt) = chrono::NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%d %H:%M:%S") {
531            return Ok(Utc.from_utc_datetime(&naive_dt));
532        }
533
534        // Try parsing pure date formats and convert to start of day
535        let date_formats = [
536            "%Y-%m-%d", // 2022-01-01
537            "%Y/%m/%d", // 2022/01/01
538            "%Y.%m.%d", // 2022.01.01
539            "%Y%m%d",   // 20220101 - must be exactly 8 digits
540        ];
541
542        for format in &date_formats {
543            if let Ok(date) = NaiveDate::parse_from_str(ts_str, format) {
544                // Additional validation for compact format to ensure it's actually a date
545                if format == &"%Y%m%d" && ts_str.len() != 8 {
546                    continue;
547                }
548                // Convert to start of day in UTC
549                if let Some(naive_datetime) = date.and_hms_opt(0, 0, 0) {
550                    return Ok(Utc.from_utc_datetime(&naive_datetime));
551                }
552            }
553        }
554
555        // Finally, try parsing as Unix timestamp (only if it's reasonable length and all digits)
556        if ts_str.len() >= 9 && ts_str.len() <= 13 && ts_str.chars().all(|c| c.is_ascii_digit()) {
557            if let Ok(timestamp) = ts_str.parse::<i64>() {
558                if let Some(dt) = Utc.timestamp_opt(timestamp, 0).single() {
559                    return Ok(dt);
560                }
561            }
562        }
563
564        Err(BrokerError::ConfigurationError(format!(
565            "Invalid timestamp format '{ts_str}'. Supported formats:\n\
566                - Unix timestamp: '1640995200'\n\
567                - RFC3339 with timezone: '2022-01-01T00:00:00+00:00', '2022-01-01T00:00:00Z', '2022-01-01T05:00:00-05:00'\n\
568                - RFC3339 without timezone: '2022-01-01T00:00:00' (assumes UTC)\n\
569                - Date with time: '2022-01-01 00:00:00'\n\
570                - Pure date: '2022-01-01', '2022/01/01', '2022.01.01', '20220101'"
571        )))
572    }
573
574    /// Validate all configuration parameters before making API calls.
575    ///
576    /// This performs the same validation that was previously done at configuration time,
577    /// but now happens just before queries are executed. Returns normalized query parameters.
578    fn validate_configuration(&self) -> Result<QueryParams, BrokerError> {
579        // Validate timestamps and normalize them
580        let mut normalized_params = self.query_params.clone();
581
582        // Apply default 30-day time window if no timestamps specified
583        // This prevents slow full-table scans and avoids returning phantom 1970 entries
584        if normalized_params.ts_start.is_none() && normalized_params.ts_end.is_none() {
585            let now = chrono::Utc::now();
586            let thirty_days_ago = now - chrono::Duration::days(30);
587            normalized_params.ts_start =
588                Some(thirty_days_ago.format("%Y-%m-%dT%H:%M:%SZ").to_string());
589            normalized_params.ts_end = Some(now.format("%Y-%m-%dT%H:%M:%SZ").to_string());
590        }
591
592        if let Some(ts) = &self.query_params.ts_start {
593            let parsed_datetime = Self::parse_timestamp(ts)?;
594            normalized_params.ts_start =
595                Some(parsed_datetime.format("%Y-%m-%dT%H:%M:%SZ").to_string());
596        }
597
598        if let Some(ts) = &self.query_params.ts_end {
599            let parsed_datetime = Self::parse_timestamp(ts)?;
600            normalized_params.ts_end =
601                Some(parsed_datetime.format("%Y-%m-%dT%H:%M:%SZ").to_string());
602        }
603
604        // Permissive collector validation: normalize only, no network I/O
605        if let Some(collector_str) = &self.query_params.collector_id {
606            let collectors: Vec<String> = collector_str
607                .split(',')
608                .map(|s| s.trim())
609                .filter(|s| !s.is_empty())
610                .map(|s| s.to_string())
611                .collect();
612
613            if collectors.is_empty() {
614                return Err(BrokerError::ConfigurationError(
615                    "Collector ID cannot be empty".to_string(),
616                ));
617            }
618
619            // Deduplicate while preserving order
620            let mut seen = HashSet::new();
621            let mut deduped = Vec::with_capacity(collectors.len());
622            for c in collectors {
623                if seen.insert(c.clone()) {
624                    deduped.push(c);
625                }
626            }
627
628            normalized_params.collector_id = Some(deduped.join(","));
629        }
630
631        // Validate project
632        if let Some(project_str) = &self.query_params.project {
633            let project_lower = project_str.to_lowercase();
634            match project_lower.as_str() {
635                "rrc" | "riperis" | "ripe_ris" | "routeviews" | "route_views" | "rv" => {
636                    // Valid project
637                }
638                _ => {
639                    return Err(BrokerError::ConfigurationError(format!(
640                        "Invalid project '{project_str}'. Valid projects are: 'riperis' (aliases: 'rrc', 'ripe_ris') or 'routeviews' (aliases: 'route_views', 'rv')"
641                    )));
642                }
643            }
644        }
645
646        // Validate data type
647        if let Some(data_type_str) = &self.query_params.data_type {
648            let data_type_lower = data_type_str.to_lowercase();
649            match data_type_lower.as_str() {
650                "rib" | "ribs" | "r" | "update" | "updates" => {
651                    // Valid data type
652                }
653                _ => {
654                    return Err(BrokerError::ConfigurationError(format!(
655                        "Invalid data type '{data_type_str}'. Valid data types are: 'rib' (aliases: 'ribs', 'r') or 'updates' (alias: 'update')"
656                    )));
657                }
658            }
659        }
660
661        // Validate page number
662        if self.query_params.page < 1 {
663            return Err(BrokerError::ConfigurationError(format!(
664                "Invalid page number {}. Page number must be >= 1",
665                self.query_params.page
666            )));
667        }
668
669        // Validate page size
670        if !(1..=100000).contains(&self.query_params.page_size) {
671            return Err(BrokerError::ConfigurationError(format!(
672                "Invalid page size {}. Page size must be between 1 and 100000",
673                self.query_params.page_size
674            )));
675        }
676
677        Ok(normalized_params)
678    }
679
680    /// Add a filter of starting timestamp.
681    ///
682    /// Supports multiple timestamp formats including Unix timestamps, RFC3339 dates, and pure dates.
683    /// Validation occurs at query time.
684    ///
685    /// # Examples
686    ///
687    /// Specify a Unix timestamp:
688    /// ```
689    /// let broker = bgpkit_broker::BgpkitBroker::new()
690    ///     .ts_start("1640995200");
691    /// ```
692    ///
693    /// Specify a RFC3339-formatted time string:
694    /// ```
695    /// let broker = bgpkit_broker::BgpkitBroker::new()
696    ///     .ts_start("2022-01-01T00:00:00Z");
697    /// ```
698    ///
699    /// Specify a pure date (defaults to start of day):
700    /// ```
701    /// let broker = bgpkit_broker::BgpkitBroker::new()
702    ///     .ts_start("2022-01-01");
703    /// ```
704    ///
705    /// Other supported formats:
706    /// ```
707    /// let broker = bgpkit_broker::BgpkitBroker::new()
708    ///     .ts_start("2022/01/01")  // slash format
709    ///     .ts_start("2022.01.01")  // dot format
710    ///     .ts_start("20220101");   // compact format
711    /// ```
712    pub fn ts_start<S: Display>(self, ts_start: S) -> Self {
713        let mut query_params = self.query_params;
714        query_params.ts_start = Some(ts_start.to_string());
715        Self {
716            broker_url: self.broker_url,
717            query_params,
718            client: self.client,
719            collector_project_map: self.collector_project_map,
720            accept_invalid_certs: self.accept_invalid_certs,
721            cache_dir: self.cache_dir,
722        }
723    }
724
725    /// Add a filter of ending timestamp.
726    ///
727    /// Supports the same multiple timestamp formats as `ts_start`.
728    /// Validation occurs at query time.
729    ///
730    /// # Examples
731    ///
732    /// Specify a Unix timestamp:
733    /// ```
734    /// let broker = bgpkit_broker::BgpkitBroker::new()
735    ///     .ts_end("1640995200");
736    /// ```
737    ///
738    /// Specify a RFC3339-formatted time string:
739    /// ```
740    /// let broker = bgpkit_broker::BgpkitBroker::new()
741    ///     .ts_end("2022-01-01T00:00:00Z");
742    /// ```
743    ///
744    /// Specify a pure date (defaults to start of day):
745    /// ```
746    /// let broker = bgpkit_broker::BgpkitBroker::new()
747    ///     .ts_end("2022-01-01");
748    /// ```
749    pub fn ts_end<S: Display>(self, ts_end: S) -> Self {
750        let mut query_params = self.query_params;
751        query_params.ts_end = Some(ts_end.to_string());
752        Self {
753            broker_url: self.broker_url,
754            client: self.client,
755            query_params,
756            collector_project_map: self.collector_project_map,
757            accept_invalid_certs: self.accept_invalid_certs,
758            cache_dir: self.cache_dir,
759        }
760    }
761
762    /// Add a filter of collector ID (e.g. `rrc00` or `route-views2`).
763    ///
764    /// See the full list of collectors [here](https://github.com/bgpkit/bgpkit-broker-backend/blob/main/deployment/full-config.json).
765    /// Validation occurs at query time.
766    ///
767    /// # Examples
768    ///
769    /// filter by single collector
770    /// ```
771    /// let broker = bgpkit_broker::BgpkitBroker::new()
772    ///     .collector_id("rrc00");
773    /// ```
774    ///
775    /// filter by multiple collector
776    /// ```
777    /// let broker = bgpkit_broker::BgpkitBroker::new()
778    ///     .collector_id("route-views2,route-views6");
779    /// ```
780    pub fn collector_id<S: Display>(self, collector_id: S) -> Self {
781        let mut query_params = self.query_params;
782        query_params.collector_id = Some(collector_id.to_string());
783        Self {
784            client: self.client,
785            broker_url: self.broker_url,
786            query_params,
787            collector_project_map: self.collector_project_map,
788            accept_invalid_certs: self.accept_invalid_certs,
789            cache_dir: self.cache_dir,
790        }
791    }
792
793    /// Add a filter of project name with validation, i.e. `riperis` or `routeviews`.
794    ///
795    /// # Examples
796    ///
797    /// ```
798    /// let broker = bgpkit_broker::BgpkitBroker::new()
799    ///     .project("riperis");
800    /// ```
801    ///
802    /// ```
803    /// let broker = bgpkit_broker::BgpkitBroker::new()
804    ///     .project("routeviews");
805    /// ```
806    pub fn project<S: Display>(self, project: S) -> Self {
807        let mut query_params = self.query_params;
808        query_params.project = Some(project.to_string());
809        Self {
810            client: self.client,
811            broker_url: self.broker_url,
812            query_params,
813            collector_project_map: self.collector_project_map,
814            accept_invalid_certs: self.accept_invalid_certs,
815            cache_dir: self.cache_dir,
816        }
817    }
818
819    /// Add filter of data type, i.e. `rib` or `updates`.
820    ///
821    /// Validation occurs at query time.
822    ///
823    /// # Examples
824    ///
825    /// ```
826    /// let broker = bgpkit_broker::BgpkitBroker::new()
827    ///     .data_type("rib");
828    /// ```
829    ///
830    /// ```
831    /// let broker = bgpkit_broker::BgpkitBroker::new()
832    ///     .data_type("updates");
833    /// ```
834    pub fn data_type<S: Display>(self, data_type: S) -> Self {
835        let mut query_params = self.query_params;
836        query_params.data_type = Some(data_type.to_string());
837        Self {
838            broker_url: self.broker_url,
839            client: self.client,
840            query_params,
841            collector_project_map: self.collector_project_map,
842            accept_invalid_certs: self.accept_invalid_certs,
843            cache_dir: self.cache_dir,
844        }
845    }
846
847    /// Change the current page number, starting from 1.
848    ///
849    /// Validation occurs at query time.
850    ///
851    /// # Examples
852    ///
853    /// Start iterating with page 2.
854    /// ```
855    /// let broker = bgpkit_broker::BgpkitBroker::new()
856    ///     .page(2);
857    /// ```
858    pub fn page(self, page: i64) -> Self {
859        let mut query_params = self.query_params;
860        query_params.page = page;
861        Self {
862            broker_url: self.broker_url,
863            client: self.client,
864            query_params,
865            collector_project_map: self.collector_project_map,
866            accept_invalid_certs: self.accept_invalid_certs,
867            cache_dir: self.cache_dir,
868        }
869    }
870
871    /// Change current page size, default 100.
872    ///
873    /// Validation occurs at query time.
874    ///
875    /// # Examples
876    ///
877    /// Set page size to 20.
878    /// ```
879    /// let broker = bgpkit_broker::BgpkitBroker::new()
880    ///     .page_size(10);
881    /// ```
882    pub fn page_size(self, page_size: i64) -> Self {
883        let mut query_params = self.query_params;
884        query_params.page_size = page_size;
885        Self {
886            broker_url: self.broker_url,
887            client: self.client,
888            query_params,
889            collector_project_map: self.collector_project_map,
890            accept_invalid_certs: self.accept_invalid_certs,
891            cache_dir: self.cache_dir,
892        }
893    }
894
895    /// Add a filter of peer IP address when listing peers.
896    ///
897    /// # Examples
898    ///
899    /// ```
900    /// let broker = bgpkit_broker::BgpkitBroker::new()
901    ///    .peers_ip("192.168.1.1".parse().unwrap());
902    /// ```
903    pub fn peers_ip(self, peer_ip: IpAddr) -> Self {
904        let mut query_params = self.query_params;
905        query_params.peers_ip = Some(peer_ip);
906        Self {
907            broker_url: self.broker_url,
908            client: self.client,
909            query_params,
910            collector_project_map: self.collector_project_map,
911            accept_invalid_certs: self.accept_invalid_certs,
912            cache_dir: self.cache_dir,
913        }
914    }
915
916    /// Add a filter of peer ASN when listing peers.
917    ///
918    /// # Examples
919    ///
920    /// ```
921    /// let broker = bgpkit_broker::BgpkitBroker::new()
922    ///    .peers_asn(64496);
923    /// ```
924    pub fn peers_asn(self, peer_asn: u32) -> Self {
925        let mut query_params = self.query_params;
926        query_params.peers_asn = Some(peer_asn);
927        Self {
928            broker_url: self.broker_url,
929            client: self.client,
930            query_params,
931            collector_project_map: self.collector_project_map,
932            accept_invalid_certs: self.accept_invalid_certs,
933            cache_dir: self.cache_dir,
934        }
935    }
936
937    /// Add a filter of peer full feed status when listing peers.
938    ///
939    /// # Examples
940    ///
941    /// ```
942    /// let broker = bgpkit_broker::BgpkitBroker::new()
943    ///   .peers_only_full_feed(true);
944    /// ```
945    pub fn peers_only_full_feed(self, peer_full_feed: bool) -> Self {
946        let mut query_params = self.query_params;
947        query_params.peers_only_full_feed = peer_full_feed;
948        Self {
949            broker_url: self.broker_url,
950            client: self.client,
951            query_params,
952            collector_project_map: self.collector_project_map,
953            accept_invalid_certs: self.accept_invalid_certs,
954            cache_dir: self.cache_dir,
955        }
956    }
957
958    /// Turn to specified page, page starting from 1.
959    ///
960    /// This works with [Self::query_single_page] function to manually paginate.
961    ///
962    /// # Examples
963    ///
964    /// Manually get the first two pages of items.
965    /// ```no_run
966    /// let mut broker = bgpkit_broker::BgpkitBroker::new();
967    /// let mut items = vec![];
968    /// items.extend(broker.query_single_page().unwrap());
969    /// broker.turn_page(2);
970    /// items.extend(broker.query_single_page().unwrap());
971    /// ```
972    pub fn turn_page(&mut self, page: i64) {
973        self.query_params.page = page;
974    }
975
976    /// Send API for a single page of items.
977    ///
978    /// # Examples
979    ///
980    /// Manually get the first page of items.
981    /// ```no_run
982    /// let broker = bgpkit_broker::BgpkitBroker::new();
983    /// let items = broker.query_single_page().unwrap();
984    /// ```
985    pub fn query_single_page(&self) -> Result<Vec<BrokerItem>, BrokerError> {
986        // Try to load from cache first
987        if let Some(cached_items) = self.load_cache() {
988            return Ok(cached_items);
989        }
990
991        let validated_params = self.validate_configuration()?;
992        let url = format!("{}/search{}", self.broker_url, validated_params);
993        log::info!("sending broker query to {}", url);
994        match self.run_files_query(url.as_str()) {
995            Ok(res) => {
996                // Save to cache if cache_dir is set
997                self.save_cache(&res.data);
998                Ok(res.data)
999            }
1000            Err(e) => Err(e),
1001        }
1002    }
1003
1004    /// Query the total count of items matching the current search criteria without fetching the items.
1005    ///
1006    /// This method is useful when you need to know how many items match your search criteria
1007    /// without downloading all the items. It performs the same validation as a regular query
1008    /// but only returns the count.
1009    ///
1010    /// # Returns
1011    /// - `Ok(i64)`: The total number of matching items
1012    /// - `Err(BrokerError)`: If the query fails or the count is missing from the response
1013    ///
1014    /// # Examples
1015    ///
1016    /// ```no_run
1017    /// use bgpkit_broker::BgpkitBroker;
1018    ///
1019    /// let broker = BgpkitBroker::new()
1020    ///     .ts_start("2024-01-01")
1021    ///     .ts_end("2024-01-02")
1022    ///     .collector_id("route-views2");
1023    ///
1024    /// let count = broker.query_total_count().unwrap();
1025    /// println!("Found {} matching items", count);
1026    /// ```
1027    pub fn query_total_count(&self) -> Result<i64, BrokerError> {
1028        let validated_params = self.validate_configuration()?;
1029        let url = format!("{}/search{}", self.broker_url, validated_params);
1030        match self.run_files_query(url.as_str()) {
1031            Ok(res) => res.total.ok_or(BrokerError::BrokerError(
1032                "count not found in response".to_string(),
1033            )),
1034            Err(e) => Err(e),
1035        }
1036    }
1037
1038    /// Check if the broker instance is healthy.
1039    ///
1040    /// # Examples
1041    ///
1042    /// ```no_run
1043    /// let broker = bgpkit_broker::BgpkitBroker::new();
1044    /// assert!(broker.health_check().is_ok())
1045    /// ```
1046    pub fn health_check(&self) -> Result<(), BrokerError> {
1047        let url = format!("{}/health", self.broker_url.trim_end_matches('/'));
1048        match self.client.get(url.as_str()).send() {
1049            Ok(response) => {
1050                if response.status() == reqwest::StatusCode::OK {
1051                    Ok(())
1052                } else {
1053                    Err(BrokerError::BrokerError(format!(
1054                        "endpoint unhealthy {}",
1055                        self.broker_url
1056                    )))
1057                }
1058            }
1059            Err(_e) => Err(BrokerError::BrokerError(format!(
1060                "endpoint unhealthy {}",
1061                self.broker_url
1062            ))),
1063        }
1064    }
1065
1066    /// Send a query to get **all** data times returned.
1067    ///
1068    /// This usually is what one needs.
1069    ///
1070    /// # Examples
1071    ///
1072    /// Get all RIB files on 2022-01-01 from route-views2.
1073    /// ```no_run
1074    /// let broker = bgpkit_broker::BgpkitBroker::new()
1075    ///     .ts_start("2022-01-01T00:00:00Z")
1076    ///     .ts_end("2022-01-01T23:59:00Z")
1077    ///     .data_type("rib")
1078    ///     .collector_id("route-views2");
1079    /// let items = broker.query().unwrap();
1080    ///
1081    /// // 1 RIB dump very 2 hours, total of 12 files for 1 day
1082    /// assert_eq!(items.len(), 12);
1083    /// ```
1084    pub fn query(&self) -> Result<Vec<BrokerItem>, BrokerError> {
1085        let mut p = self.validate_configuration()?;
1086
1087        let mut items = vec![];
1088        loop {
1089            let url = format!("{}/search{}", self.broker_url, p);
1090
1091            let res_items = self.run_files_query(url.as_str())?.data;
1092
1093            let items_count = res_items.len() as i64;
1094
1095            if items_count == 0 {
1096                // reaches the end
1097                break;
1098            }
1099
1100            items.extend(res_items);
1101            let cur_page = p.page;
1102            p = p.page(cur_page + 1);
1103
1104            if items_count < p.page_size {
1105                // reaches the end
1106                break;
1107            }
1108        }
1109        Ok(items)
1110    }
1111
1112    /// Send a query to get the **latest** data for each collector.
1113    ///
1114    /// The returning result is structured as a vector of [CollectorLatestItem] objects.
1115    ///
1116    /// # Examples
1117    ///
1118    /// ```no_run
1119    /// let broker = bgpkit_broker::BgpkitBroker::new();
1120    /// let latest_items = broker.latest().unwrap();
1121    /// for item in &latest_items {
1122    ///     println!("{}", item);
1123    /// }
1124    /// ```
1125    pub fn latest(&self) -> Result<Vec<BrokerItem>, BrokerError> {
1126        let latest_query_url = format!("{}/latest", self.broker_url);
1127        let mut items = match self.client.get(latest_query_url.as_str()).send() {
1128            Ok(response) => match response.json::<CollectorLatestResult>() {
1129                Ok(result) => result.data,
1130                Err(_) => {
1131                    return Err(BrokerError::BrokerError(
1132                        "Error parsing response".to_string(),
1133                    ));
1134                }
1135            },
1136            Err(e) => {
1137                return Err(BrokerError::BrokerError(format!(
1138                    "Unable to connect to the URL ({latest_query_url}): {e}"
1139                )));
1140            }
1141        };
1142
1143        items.retain(|item| {
1144            let mut matches = true;
1145            if let Some(project) = &self.query_params.project {
1146                match project.to_lowercase().as_str() {
1147                    "rrc" | "riperis" | "ripe_ris" => {
1148                        matches = self
1149                            .collector_project_map
1150                            .get(&item.collector_id)
1151                            .cloned()
1152                            .unwrap_or_default()
1153                            .as_str()
1154                            == "riperis";
1155                    }
1156                    "routeviews" | "route_views" | "rv" => {
1157                        matches = self
1158                            .collector_project_map
1159                            .get(&item.collector_id)
1160                            .cloned()
1161                            .unwrap_or_default()
1162                            .as_str()
1163                            == "routeviews";
1164                    }
1165                    _ => {}
1166                }
1167            }
1168
1169            if let Some(data_type) = &self.query_params.data_type {
1170                match data_type.to_lowercase().as_str() {
1171                    "rib" | "ribs" | "r" if !item.is_rib() => {
1172                        // if not RIB file, not match
1173                        matches = false
1174                    }
1175                    "update" | "updates" if item.is_rib() => {
1176                        // if is RIB file, not match
1177                        matches = false
1178                    }
1179                    _ => {}
1180                }
1181            }
1182
1183            if let Some(collector_id) = &self.query_params.collector_id {
1184                let wanted: HashSet<&str> = collector_id
1185                    .split(',')
1186                    .map(|s| s.trim())
1187                    .filter(|s| !s.is_empty())
1188                    .collect();
1189
1190                if !wanted.contains(item.collector_id.as_str()) {
1191                    return false;
1192                }
1193            }
1194
1195            matches
1196        });
1197
1198        Ok(items)
1199    }
1200
1201    /// Get the most recent information for collector peers.
1202    ///
1203    /// The returning result is structured as a vector of [BrokerPeer] objects.
1204    ///
1205    /// # Examples
1206    ///
1207    /// ## Get all peers
1208    ///
1209    /// ```no_run
1210    /// let broker = bgpkit_broker::BgpkitBroker::new();
1211    /// let peers = broker.get_peers().unwrap();
1212    /// for peer in &peers {
1213    ///     println!("{:?}", peer);
1214    /// }
1215    /// ```
1216    ///
1217    /// ## Get peers from a specific collector
1218    ///
1219    /// ```no_run
1220    /// let broker = bgpkit_broker::BgpkitBroker::new()
1221    ///    .collector_id("route-views2");
1222    /// let peers = broker.get_peers().unwrap();
1223    /// for peer in &peers {
1224    ///    println!("{:?}", peer);
1225    /// }
1226    /// ```
1227    ///
1228    /// ## Get peers from a specific ASN
1229    ///
1230    /// ```no_run
1231    /// let broker = bgpkit_broker::BgpkitBroker::new()
1232    ///   .peers_asn(64496);
1233    /// let peers = broker.get_peers().unwrap();
1234    /// for peer in &peers {
1235    ///    println!("{:?}", peer);
1236    /// }
1237    /// ```
1238    ///
1239    /// ## Get peers from a specific IP address
1240    ///
1241    /// ```no_run
1242    /// let broker = bgpkit_broker::BgpkitBroker::new()
1243    ///   .peers_ip("192.168.1.1".parse().unwrap());
1244    /// let peers = broker.get_peers().unwrap();
1245    /// for peer in &peers {
1246    ///   println!("{:?}", peer);
1247    /// }
1248    /// ```
1249    ///
1250    /// ## Get peers with full feed
1251    ///
1252    /// ```no_run
1253    /// let broker = bgpkit_broker::BgpkitBroker::new()
1254    ///  .peers_only_full_feed(true);
1255    /// let peers = broker.get_peers().unwrap();
1256    /// for peer in &peers {
1257    ///     println!("{:?}", peer);
1258    /// }
1259    /// ```
1260    ///
1261    /// ## Get peers from a specific collector with full feed
1262    ///
1263    /// ```no_run
1264    /// let broker = bgpkit_broker::BgpkitBroker::new()
1265    ///  .collector_id("route-views2")
1266    /// .peers_only_full_feed(true);
1267    /// let peers = broker.get_peers().unwrap();
1268    /// for peer in &peers {
1269    ///    println!("{:?}", peer);
1270    /// }
1271    /// ```
1272    pub fn get_peers(&self) -> Result<Vec<BrokerPeer>, BrokerError> {
1273        let mut url = format!("{}/peers", self.broker_url);
1274        let mut param_strings = vec![];
1275        if let Some(ip) = &self.query_params.peers_ip {
1276            param_strings.push(format!("ip={ip}"));
1277        }
1278        if let Some(asn) = &self.query_params.peers_asn {
1279            param_strings.push(format!("asn={asn}"));
1280        }
1281        if self.query_params.peers_only_full_feed {
1282            param_strings.push("full_feed=true".to_string());
1283        }
1284        if let Some(collector_id) = &self.query_params.collector_id {
1285            param_strings.push(format!("collector={collector_id}"));
1286        }
1287        if !param_strings.is_empty() {
1288            let param_string = param_strings.join("&");
1289            url = format!("{url}?{param_string}");
1290        }
1291
1292        let peers = match self.client.get(url.as_str()).send() {
1293            Ok(response) => match response.json::<BrokerPeersResult>() {
1294                Ok(result) => result.data,
1295                Err(_) => {
1296                    return Err(BrokerError::BrokerError(
1297                        "Error parsing response".to_string(),
1298                    ));
1299                }
1300            },
1301            Err(e) => {
1302                return Err(BrokerError::BrokerError(format!(
1303                    "Unable to connect to the URL ({url}): {e}"
1304                )));
1305            }
1306        };
1307        Ok(peers)
1308    }
1309
1310    fn run_files_query(&self, url: &str) -> Result<BrokerQueryResult, BrokerError> {
1311        log::info!("sending broker query to {}", url);
1312        match self.client.get(url).send() {
1313            Ok(res) => {
1314                let status = res.status();
1315                if !status.is_success() {
1316                    // A non-2xx body carries the API's own explanation (for
1317                    // example "All deployment instances are down"). Decoding it
1318                    // as a success payload only produced a serde message such as
1319                    // "error decoding response body", which hid the real cause.
1320                    let body = res.text().unwrap_or_default();
1321                    let body = body.trim();
1322                    let body: String = body.chars().take(512).collect();
1323                    return Err(BrokerError::BrokerError(format!(
1324                        "broker API returned HTTP {status}: {body}"
1325                    )));
1326                }
1327                match res.json::<BrokerQueryResult>() {
1328                    Ok(res) => {
1329                        if let Some(e) = res.error {
1330                            Err(BrokerError::BrokerError(e))
1331                        } else {
1332                            Ok(res)
1333                        }
1334                    }
1335                    Err(e) => {
1336                        // json decoding error. most likely the service returns an error message without
1337                        // `data` field.
1338                        Err(BrokerError::BrokerError(e.to_string()))
1339                    }
1340                }
1341            }
1342            Err(e) => Err(BrokerError::from(e)),
1343        }
1344    }
1345}
1346
1347/// Iterator for BGPKIT Broker that iterates through one [BrokerItem] at a time.
1348///
1349/// The [IntoIterator] trait is implemented for both the struct and the reference, so that you can
1350/// either iterate through items by taking the ownership of the broker, or use the reference to broker
1351/// to iterate.
1352///
1353/// ```no_run
1354/// use bgpkit_broker::{BgpkitBroker, BrokerItem};
1355///
1356/// let mut broker = BgpkitBroker::new()
1357///     .ts_start("1634693400")
1358///     .ts_end("1634693400")
1359///     .page_size(10)
1360///     .page(2);
1361///
1362/// // create iterator from reference (so that you can reuse the broker object)
1363/// // same as `&broker.into_intr()`
1364/// for item in &broker {
1365///     println!("{}", item);
1366/// }
1367///
1368/// // create iterator from the broker object (taking ownership)
1369/// let items = broker.into_iter().collect::<Vec<BrokerItem>>();
1370///
1371/// assert_eq!(items.len(), 43);
1372/// ```
1373pub struct BrokerItemIterator {
1374    broker: BgpkitBroker,
1375    cached_items: Vec<BrokerItem>,
1376    first_run: bool,
1377}
1378
1379impl BrokerItemIterator {
1380    pub fn new(broker: BgpkitBroker) -> BrokerItemIterator {
1381        BrokerItemIterator {
1382            broker,
1383            cached_items: vec![],
1384            first_run: true,
1385        }
1386    }
1387}
1388
1389impl Iterator for BrokerItemIterator {
1390    type Item = BrokerItem;
1391
1392    fn next(&mut self) -> Option<Self::Item> {
1393        // if we have cached items, simply pop and return
1394        if let Some(item) = self.cached_items.pop() {
1395            return Some(item);
1396        }
1397
1398        // no more cached items, refill cache by one more broker query
1399        if self.first_run {
1400            // if it's the first time running, do not change page, and switch the flag.
1401            self.first_run = false;
1402        } else {
1403            // if it's not the first time running, add page number by one.
1404            self.broker.query_params.page += 1;
1405        }
1406
1407        // query the current page
1408        let items = match self.broker.query_single_page() {
1409            Ok(i) => i,
1410            Err(_) => return None,
1411        };
1412
1413        if items.is_empty() {
1414            // break out the iteration
1415            return None;
1416        } else {
1417            // fill the cache
1418            self.cached_items = items;
1419            self.cached_items.reverse();
1420        }
1421
1422        #[allow(clippy::unwrap_used)]
1423        Some(self.cached_items.pop().unwrap())
1424    }
1425}
1426
1427impl IntoIterator for BgpkitBroker {
1428    type Item = BrokerItem;
1429    type IntoIter = BrokerItemIterator;
1430
1431    fn into_iter(self) -> Self::IntoIter {
1432        BrokerItemIterator::new(self)
1433    }
1434}
1435
1436impl IntoIterator for &BgpkitBroker {
1437    type Item = BrokerItem;
1438    type IntoIter = BrokerItemIterator;
1439
1440    fn into_iter(self) -> Self::IntoIter {
1441        BrokerItemIterator::new(self.clone())
1442    }
1443}
1444
1445#[cfg(test)]
1446mod tests {
1447    use super::*;
1448
1449    /// A non-2xx answer must surface the API's own error text, not a JSON
1450    /// decode failure that hides the cause.
1451    #[test]
1452    fn test_http_error_body_is_surfaced() {
1453        use std::io::{BufRead, BufReader, Write};
1454        use std::net::TcpListener;
1455
1456        let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port");
1457        let addr = listener.local_addr().expect("local addr");
1458        let server = std::thread::spawn(move || {
1459            if let Ok((mut stream, _)) = listener.accept() {
1460                let mut reader = BufReader::new(stream.try_clone().expect("clone stream"));
1461                let mut request_line = String::new();
1462                let _ = reader.read_line(&mut request_line);
1463                let body = r#"{"error":"Failed to fetch search results","message":"All deployment instances are down"}"#;
1464                let response = format!(
1465                    "HTTP/1.1 500 Internal Server Error\r\n\
1466                     Content-Type: application/json\r\n\
1467                     Content-Length: {}\r\n\
1468                     Connection: close\r\n\r\n{}",
1469                    body.len(),
1470                    body
1471                );
1472                let _ = stream.write_all(response.as_bytes());
1473                let _ = stream.flush();
1474            }
1475        });
1476
1477        let broker = BgpkitBroker::new()
1478            .broker_url(format!("http://{}", addr))
1479            .ts_start("1634693400")
1480            .ts_end("1634693400");
1481        let err = broker
1482            .query_single_page()
1483            .expect_err("HTTP 500 must be reported as an error");
1484        let msg = err.to_string();
1485        assert!(msg.contains("HTTP 500"), "unexpected message: {msg}");
1486        assert!(
1487            msg.contains("All deployment instances are down"),
1488            "server error body was dropped: {msg}"
1489        );
1490
1491        let _ = server.join();
1492    }
1493
1494    #[test]
1495    fn test_query() {
1496        let broker = BgpkitBroker::new()
1497            .ts_start("1634693400")
1498            .ts_end("1634693400");
1499        let res = broker.query();
1500        assert!(&res.is_ok());
1501        let data = res.unwrap();
1502        assert!(!data.is_empty());
1503    }
1504
1505    #[test]
1506    fn test_network_error() {
1507        let broker = BgpkitBroker::new().broker_url("https://api.broker.example.com/v2");
1508        let res = broker.query();
1509        // when testing a must-fail query, you could use `matches!` macro to do so
1510        assert!(res.is_err());
1511        assert!(matches!(res.err(), Some(BrokerError::NetworkError(_))));
1512    }
1513
1514    #[test]
1515    fn test_broker_error() {
1516        let broker = BgpkitBroker::new().page(-1);
1517        let result = broker.query();
1518        assert!(result.is_err());
1519        assert!(matches!(
1520            result.err(),
1521            Some(BrokerError::ConfigurationError(_))
1522        ));
1523    }
1524
1525    #[test]
1526    fn test_query_all() {
1527        let broker = BgpkitBroker::new()
1528            .ts_start("1634693400")
1529            .ts_end("1634693400")
1530            .page_size(100);
1531        let res = broker.query();
1532        assert!(res.is_ok());
1533        assert!(res.ok().unwrap().len() >= 54);
1534    }
1535
1536    #[test]
1537    fn test_iterator() {
1538        let broker = BgpkitBroker::new()
1539            .ts_start("1634693400")
1540            .ts_end("1634693400");
1541        assert!(broker.into_iter().count() >= 54);
1542    }
1543
1544    #[test]
1545    fn test_filters() {
1546        let broker = BgpkitBroker::new()
1547            .ts_start("1634693400")
1548            .ts_end("1634693400");
1549        let items = broker.query().unwrap();
1550        assert!(items.len() >= 54);
1551
1552        let broker = BgpkitBroker::new()
1553            .ts_start("1634693400")
1554            .ts_end("1634693400")
1555            .collector_id("rrc00");
1556        let items = broker.query().unwrap();
1557        assert_eq!(items.len(), 1);
1558
1559        let broker = BgpkitBroker::new()
1560            .ts_start("1634693400")
1561            .ts_end("1634693400")
1562            .project("riperis");
1563        let items = broker.query().unwrap();
1564        assert_eq!(items.len(), 23);
1565    }
1566
1567    #[test]
1568    fn test_latest() {
1569        let broker = BgpkitBroker::new();
1570        let items = broker.latest().unwrap();
1571        assert!(items.len() >= 125);
1572
1573        let broker = BgpkitBroker::new().project("routeviews".to_string());
1574        let items = broker.latest().unwrap();
1575        assert!(!items.is_empty());
1576        assert!(items
1577            .iter()
1578            .all(|item| !item.collector_id.starts_with("rrc")));
1579
1580        let broker = BgpkitBroker::new().project("riperis".to_string());
1581        let items = broker.latest().unwrap();
1582        assert!(!items.is_empty());
1583        assert!(items
1584            .iter()
1585            .all(|item| item.collector_id.starts_with("rrc")));
1586
1587        let broker = BgpkitBroker::new().data_type("rib".to_string());
1588        let items = broker.latest().unwrap();
1589        assert!(!items.is_empty());
1590        assert!(items.iter().all(|item| item.is_rib()));
1591
1592        let broker = BgpkitBroker::new().data_type("update".to_string());
1593        let items = broker.latest().unwrap();
1594        assert!(!items.is_empty());
1595        assert!(items.iter().all(|item| !item.is_rib()));
1596
1597        let broker = BgpkitBroker::new().collector_id("rrc00".to_string());
1598        let items = broker.latest().unwrap();
1599        assert!(!items.is_empty());
1600        assert!(items
1601            .iter()
1602            .all(|item| item.collector_id.as_str() == "rrc00"));
1603        assert_eq!(items.len(), 2);
1604    }
1605
1606    #[test]
1607    fn test_latest_no_ssl() {
1608        let broker = BgpkitBroker::new().accept_invalid_certs();
1609        let items = broker.latest().unwrap();
1610        assert!(items.len() >= 125);
1611    }
1612
1613    #[test]
1614    fn test_health_check() {
1615        let broker = BgpkitBroker::new();
1616        let res = broker.health_check();
1617        assert!(res.is_ok());
1618    }
1619
1620    #[test]
1621    fn test_peers() {
1622        let broker = BgpkitBroker::new();
1623        let all_peers = broker.get_peers().unwrap();
1624        assert!(!all_peers.is_empty());
1625        let first_peer = all_peers.first().unwrap();
1626        let first_ip = first_peer.ip;
1627        let first_asn = first_peer.asn;
1628
1629        let broker = BgpkitBroker::new().peers_ip(first_ip);
1630        let peers = broker.get_peers().unwrap();
1631        assert!(!peers.is_empty());
1632
1633        let broker = BgpkitBroker::new().peers_asn(first_asn);
1634        let peers = broker.get_peers().unwrap();
1635        assert!(!peers.is_empty());
1636
1637        let broker = BgpkitBroker::new().peers_only_full_feed(true);
1638        let full_feed_peers = broker.get_peers().unwrap();
1639        assert!(!full_feed_peers.is_empty());
1640        assert!(full_feed_peers.len() < all_peers.len());
1641
1642        let broker = BgpkitBroker::new().collector_id("rrc00");
1643        let rrc_peers = broker.get_peers().unwrap();
1644        assert!(!rrc_peers.is_empty());
1645        assert!(rrc_peers.iter().all(|peer| peer.collector == "rrc00"));
1646
1647        let broker = BgpkitBroker::new().collector_id("rrc00,route-views2");
1648        let rrc_rv_peers = broker.get_peers().unwrap();
1649        assert!(!rrc_rv_peers.is_empty());
1650        assert!(rrc_rv_peers
1651            .iter()
1652            .any(|peer| peer.collector == "rrc00" || peer.collector == "route-views2"));
1653
1654        assert!(rrc_rv_peers.len() > rrc_peers.len());
1655    }
1656
1657    #[test]
1658    fn test_timestamp_parsing_unix() {
1659        let broker = BgpkitBroker::new();
1660
1661        // Valid Unix timestamps - configuration succeeds, normalization happens at query time
1662        let result = broker.clone().ts_start("1640995200");
1663        // Raw input is stored during configuration
1664        assert_eq!(result.query_params.ts_start, Some("1640995200".to_string()));
1665
1666        let result = broker.clone().ts_end("1640995200");
1667        assert_eq!(result.query_params.ts_end, Some("1640995200".to_string()));
1668    }
1669
1670    #[test]
1671    fn test_timestamp_parsing_rfc3339() {
1672        let broker = BgpkitBroker::new();
1673
1674        // RFC3339 with Z - raw input stored during configuration
1675        let result = broker.clone().ts_start("2022-01-01T00:00:00Z");
1676        assert_eq!(
1677            result.query_params.ts_start,
1678            Some("2022-01-01T00:00:00Z".to_string())
1679        );
1680
1681        // RFC3339 without Z - raw input stored during configuration
1682        let result = broker.clone().ts_start("2022-01-01T12:30:45");
1683        assert_eq!(
1684            result.query_params.ts_start,
1685            Some("2022-01-01T12:30:45".to_string())
1686        );
1687
1688        // Date with time format - raw input stored during configuration
1689        let result = broker.clone().ts_end("2022-01-01 12:30:45");
1690        assert_eq!(
1691            result.query_params.ts_end,
1692            Some("2022-01-01 12:30:45".to_string())
1693        );
1694    }
1695
1696    #[test]
1697    fn test_timestamp_parsing_pure_dates() {
1698        let broker = BgpkitBroker::new();
1699
1700        // Standard date format - raw input stored during configuration
1701        let result = broker.clone().ts_start("2022-01-01");
1702        assert_eq!(result.query_params.ts_start, Some("2022-01-01".to_string()));
1703
1704        // Slash format
1705        let result = broker.clone().ts_start("2022/01/01");
1706        assert_eq!(result.query_params.ts_start, Some("2022/01/01".to_string()));
1707
1708        // Dot format
1709        let result = broker.clone().ts_end("2022.01.01");
1710        assert_eq!(result.query_params.ts_end, Some("2022.01.01".to_string()));
1711
1712        // Compact format
1713        let result = broker.clone().ts_end("20220101");
1714        assert_eq!(result.query_params.ts_end, Some("20220101".to_string()));
1715    }
1716
1717    #[test]
1718    fn test_timestamp_parsing_whitespace() {
1719        let broker = BgpkitBroker::new();
1720
1721        // Test that raw input with whitespace is stored during configuration
1722        let result = broker.clone().ts_start("  2022-01-01  ");
1723        assert_eq!(
1724            result.query_params.ts_start,
1725            Some("  2022-01-01  ".to_string())
1726        );
1727
1728        let result = broker.clone().ts_end("\t1640995200\n");
1729        assert_eq!(
1730            result.query_params.ts_end,
1731            Some("\t1640995200\n".to_string())
1732        );
1733    }
1734
1735    #[test]
1736    fn test_timestamp_parsing_errors() {
1737        let broker = BgpkitBroker::new();
1738
1739        // Invalid format - error occurs at query time
1740        let broker_with_invalid = broker.clone().ts_start("invalid-timestamp");
1741        let result = broker_with_invalid.query();
1742        assert!(result.is_err());
1743        assert!(matches!(
1744            result.err(),
1745            Some(BrokerError::ConfigurationError(_))
1746        ));
1747
1748        // Invalid date - error occurs at query time
1749        let broker_with_invalid = broker.clone().ts_end("2022-13-01");
1750        let result = broker_with_invalid.query();
1751        assert!(result.is_err());
1752        assert!(matches!(
1753            result.err(),
1754            Some(BrokerError::ConfigurationError(_))
1755        ));
1756
1757        // Invalid compact date - error occurs at query time
1758        let broker_with_invalid = broker.clone().ts_start("20221301");
1759        let result = broker_with_invalid.query();
1760        assert!(result.is_err());
1761        assert!(matches!(
1762            result.err(),
1763            Some(BrokerError::ConfigurationError(_))
1764        ));
1765
1766        // Partially valid format - error occurs at query time
1767        let broker_with_invalid = broker.clone().ts_start("2022-01");
1768        let result = broker_with_invalid.query();
1769        assert!(result.is_err());
1770        assert!(matches!(
1771            result.err(),
1772            Some(BrokerError::ConfigurationError(_))
1773        ));
1774    }
1775
1776    #[test]
1777    fn test_parse_timestamp_direct() {
1778        use chrono::{NaiveDate, NaiveDateTime};
1779
1780        // Test the parse_timestamp function directly - it now returns DateTime<Utc>
1781
1782        // Unix timestamp
1783        let expected_unix = Utc.timestamp_opt(1640995200, 0).single().unwrap();
1784        assert_eq!(
1785            BgpkitBroker::parse_timestamp("1640995200").unwrap(),
1786            expected_unix
1787        );
1788
1789        // RFC3339 formats
1790        let expected_rfc3339_z = Utc.from_utc_datetime(
1791            &NaiveDateTime::parse_from_str("2022-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S").unwrap(),
1792        );
1793        assert_eq!(
1794            BgpkitBroker::parse_timestamp("2022-01-01T00:00:00Z").unwrap(),
1795            expected_rfc3339_z
1796        );
1797
1798        let expected_rfc3339_no_z = Utc.from_utc_datetime(
1799            &NaiveDateTime::parse_from_str("2022-01-01T12:30:45", "%Y-%m-%dT%H:%M:%S").unwrap(),
1800        );
1801        assert_eq!(
1802            BgpkitBroker::parse_timestamp("2022-01-01T12:30:45").unwrap(),
1803            expected_rfc3339_no_z
1804        );
1805
1806        let expected_space_format = Utc.from_utc_datetime(
1807            &NaiveDateTime::parse_from_str("2022-01-01 12:30:45", "%Y-%m-%d %H:%M:%S").unwrap(),
1808        );
1809        assert_eq!(
1810            BgpkitBroker::parse_timestamp("2022-01-01 12:30:45").unwrap(),
1811            expected_space_format
1812        );
1813
1814        // Pure date formats (all convert to start of day in UTC)
1815        let expected_date = Utc.from_utc_datetime(
1816            &NaiveDate::from_ymd_opt(2022, 1, 1)
1817                .unwrap()
1818                .and_hms_opt(0, 0, 0)
1819                .unwrap(),
1820        );
1821        assert_eq!(
1822            BgpkitBroker::parse_timestamp("2022-01-01").unwrap(),
1823            expected_date
1824        );
1825        assert_eq!(
1826            BgpkitBroker::parse_timestamp("2022/01/01").unwrap(),
1827            expected_date
1828        );
1829        assert_eq!(
1830            BgpkitBroker::parse_timestamp("2022.01.01").unwrap(),
1831            expected_date
1832        );
1833        assert_eq!(
1834            BgpkitBroker::parse_timestamp("20220101").unwrap(),
1835            expected_date
1836        );
1837
1838        // Test timezone formats - these should now work
1839        let result_plus_tz = BgpkitBroker::parse_timestamp("2022-01-01T00:00:00+00:00").unwrap();
1840        assert_eq!(result_plus_tz, expected_date);
1841        println!("✓ +00:00 timezone format works");
1842
1843        // Test timezone conversion: 2022-01-01T05:00:00-05:00 = 2022-01-01T10:00:00Z
1844        let result_minus_tz = BgpkitBroker::parse_timestamp("2022-01-01T05:00:00-05:00").unwrap();
1845        let expected_10am = Utc.with_ymd_and_hms(2022, 1, 1, 10, 0, 0).unwrap();
1846        assert_eq!(result_minus_tz, expected_10am);
1847        println!("✓ -05:00 timezone format works (05:00-05:00 = 10:00Z)");
1848
1849        // Error cases
1850        assert!(BgpkitBroker::parse_timestamp("invalid").is_err());
1851        assert!(BgpkitBroker::parse_timestamp("2022-13-01").is_err());
1852        assert!(BgpkitBroker::parse_timestamp("2022-01").is_err());
1853    }
1854
1855    #[test]
1856    fn test_collector_id_validation() {
1857        let broker = BgpkitBroker::new();
1858
1859        // Valid single collector - no error at validation time
1860        let broker_valid = broker.clone().collector_id("rrc00");
1861        let result = broker_valid.validate_configuration();
1862        assert!(result.is_ok());
1863
1864        // Valid multiple collectors - no error at validation time
1865        let broker_valid = broker.clone().collector_id("rrc00,route-views2");
1866        let result = broker_valid.validate_configuration();
1867        assert!(result.is_ok());
1868
1869        // Unknown collector should be allowed (permissive behavior)
1870        let broker_unknown = broker.clone().collector_id("brand-new-collector");
1871        let result = broker_unknown.validate_configuration();
1872        assert!(result.is_ok());
1873
1874        // Mixed known and unknown collectors should be allowed
1875        let broker_mixed = broker.clone().collector_id("rrc00,brand-new-collector");
1876        let result = broker_mixed.validate_configuration();
1877        assert!(result.is_ok());
1878
1879        // Empty/whitespace-only should error
1880        let broker_empty = broker.clone().collector_id(", ,  ,");
1881        let result = broker_empty.validate_configuration();
1882        assert!(result.is_err());
1883        assert!(matches!(
1884            result.err(),
1885            Some(BrokerError::ConfigurationError(_))
1886        ));
1887    }
1888
1889    #[test]
1890    fn test_project_validation() {
1891        let broker = BgpkitBroker::new();
1892
1893        // Valid projects - no error at configuration time
1894        let broker_valid = broker.clone().project("riperis");
1895        let result = broker_valid.validate_configuration();
1896        assert!(result.is_ok());
1897
1898        let broker_valid = broker.clone().project("routeviews");
1899        let result = broker_valid.validate_configuration();
1900        assert!(result.is_ok());
1901
1902        // Valid aliases - no error at configuration time
1903        let broker_valid = broker.clone().project("rrc");
1904        let result = broker_valid.validate_configuration();
1905        assert!(result.is_ok());
1906
1907        let broker_valid = broker.clone().project("rv");
1908        let result = broker_valid.validate_configuration();
1909        assert!(result.is_ok());
1910
1911        // Invalid project - error occurs at validation
1912        let broker_invalid = broker.clone().project("invalid-project");
1913        let result = broker_invalid.validate_configuration();
1914        assert!(result.is_err());
1915        assert!(matches!(
1916            result.err(),
1917            Some(BrokerError::ConfigurationError(_))
1918        ));
1919    }
1920
1921    #[test]
1922    fn test_data_type_validation() {
1923        let broker = BgpkitBroker::new();
1924
1925        // Valid data types - no error at configuration time
1926        let broker_valid = broker.clone().data_type("rib");
1927        let result = broker_valid.validate_configuration();
1928        assert!(result.is_ok());
1929
1930        let broker_valid = broker.clone().data_type("updates");
1931        let result = broker_valid.validate_configuration();
1932        assert!(result.is_ok());
1933
1934        // Valid aliases - no error at configuration time
1935        let broker_valid = broker.clone().data_type("ribs");
1936        let result = broker_valid.validate_configuration();
1937        assert!(result.is_ok());
1938
1939        let broker_valid = broker.clone().data_type("update");
1940        let result = broker_valid.validate_configuration();
1941        assert!(result.is_ok());
1942
1943        // Invalid data type - error occurs at validation
1944        let broker_invalid = broker.clone().data_type("invalid-type");
1945        let result = broker_invalid.validate_configuration();
1946        assert!(result.is_err());
1947        assert!(matches!(
1948            result.err(),
1949            Some(BrokerError::ConfigurationError(_))
1950        ));
1951    }
1952
1953    #[test]
1954    fn test_page_validation() {
1955        let broker = BgpkitBroker::new();
1956
1957        // Valid page number - no error at configuration time
1958        let broker_valid = broker.clone().page(1);
1959        let result = broker_valid.validate_configuration();
1960        assert!(result.is_ok());
1961
1962        let broker_valid = broker.clone().page(100);
1963        let result = broker_valid.validate_configuration();
1964        assert!(result.is_ok());
1965
1966        // Invalid page number - error occurs at validation
1967        let broker_invalid = broker.clone().page(0);
1968        let result = broker_invalid.validate_configuration();
1969        assert!(result.is_err());
1970        assert!(matches!(
1971            result.err(),
1972            Some(BrokerError::ConfigurationError(_))
1973        ));
1974    }
1975
1976    #[test]
1977    fn test_page_size_validation() {
1978        let broker = BgpkitBroker::new();
1979
1980        // Valid page sizes - no error at configuration time
1981        let broker_valid = broker.clone().page_size(1);
1982        let result = broker_valid.validate_configuration();
1983        assert!(result.is_ok());
1984
1985        let broker_valid = broker.clone().page_size(100);
1986        let result = broker_valid.validate_configuration();
1987        assert!(result.is_ok());
1988
1989        let broker_valid = broker.clone().page_size(100000);
1990        let result = broker_valid.validate_configuration();
1991        assert!(result.is_ok());
1992
1993        // Invalid page sizes - error occurs at validation
1994        let broker_invalid = broker.clone().page_size(0);
1995        let result = broker_invalid.validate_configuration();
1996        assert!(result.is_err());
1997        assert!(matches!(
1998            result.err(),
1999            Some(BrokerError::ConfigurationError(_))
2000        ));
2001
2002        let broker_invalid = broker.clone().page_size(100001);
2003        let result = broker_invalid.validate_configuration();
2004        assert!(result.is_err());
2005        assert!(matches!(
2006            result.err(),
2007            Some(BrokerError::ConfigurationError(_))
2008        ));
2009    }
2010
2011    #[test]
2012    fn test_method_chaining() {
2013        let broker = BgpkitBroker::new()
2014            .ts_start("1634693400")
2015            .ts_end("1634693400")
2016            .collector_id("rrc00")
2017            .project("riperis")
2018            .data_type("rib")
2019            .page(1)
2020            .page_size(10);
2021
2022        // Raw input is stored during configuration
2023        assert_eq!(broker.query_params.ts_start, Some("1634693400".to_string()));
2024        assert_eq!(broker.query_params.ts_end, Some("1634693400".to_string()));
2025        assert_eq!(broker.query_params.collector_id, Some("rrc00".to_string()));
2026        assert_eq!(broker.query_params.project, Some("riperis".to_string()));
2027        assert_eq!(broker.query_params.data_type, Some("rib".to_string()));
2028        assert_eq!(broker.query_params.page, 1);
2029        assert_eq!(broker.query_params.page_size, 10);
2030    }
2031
2032    #[test]
2033    fn test_broker_url_falls_back_to_default() {
2034        assert_eq!(normalize_broker_url(None), DEFAULT_BROKER_URL);
2035        assert_eq!(
2036            normalize_broker_url(Some(String::new())),
2037            DEFAULT_BROKER_URL
2038        );
2039        assert_eq!(
2040            normalize_broker_url(Some("   ".to_string())),
2041            DEFAULT_BROKER_URL
2042        );
2043    }
2044
2045    #[test]
2046    fn test_broker_url_is_normalized() {
2047        assert_eq!(
2048            normalize_broker_url(Some(" https://broker.example.com/v2/ ".to_string())),
2049            "https://broker.example.com/v2"
2050        );
2051        assert_eq!(
2052            normalize_broker_url(Some("https://broker.example.com/v2////".to_string())),
2053            "https://broker.example.com/v2"
2054        );
2055    }
2056}