1#![doc(
203 html_logo_url = "https://raw.githubusercontent.com/bgpkit/assets/main/logos/icon-transparent.png",
204 html_favicon_url = "https://raw.githubusercontent.com/bgpkit/assets/main/logos/favicon.ico"
205)]
206#![allow(unknown_lints)]
207
208mod collector;
209#[cfg(feature = "cli")]
210pub mod config;
211#[cfg(feature = "cli")]
212pub mod crawler;
213#[cfg(feature = "backend")]
214pub mod db;
215mod error;
216mod item;
217#[cfg(feature = "nats")]
218pub mod notifier;
219mod peer;
220mod query;
221mod shortcuts;
222#[cfg(feature = "sse")]
223mod sse;
224
225use crate::collector::DEFAULT_COLLECTORS_CONFIG;
226use crate::peer::BrokerPeersResult;
227use crate::query::{BrokerQueryResult, CollectorLatestResult};
228use chrono::{DateTime, NaiveDate, TimeZone, Utc};
229pub use collector::{load_collectors, Collector};
230
231#[cfg(feature = "cli")]
232pub use config::{BrokerConfig, DatabaseTarget};
233#[cfg(feature = "cli")]
234pub use crawler::crawl_collector;
235#[cfg(feature = "cli")]
236pub use db::DatabaseBackend;
237#[cfg(feature = "backend")]
238pub use db::{LocalBrokerDb, PostgresDb, UpdatesMeta, DEFAULT_PAGE_SIZE};
239pub use error::BrokerError;
240pub use item::BrokerItem;
241pub use peer::BrokerPeer;
242pub use query::{QueryParams, SortOrder};
243pub use shortcuts::SnapshotFiles;
244#[cfg(feature = "sse")]
245pub use sse::{BrokerItemSubscription, SseSubscriptionOptions};
246use std::collections::{HashMap, HashSet};
247use std::fmt::Display;
248use std::net::IpAddr;
249use std::path::PathBuf;
250
251const SDK_USER_AGENT: &str = concat!("bgpkit-broker/", env!("CARGO_PKG_VERSION"));
252
253#[derive(Clone)]
257pub struct BgpkitBroker {
258 pub broker_url: String,
259 pub query_params: QueryParams,
260 client: reqwest::blocking::Client,
261 collector_project_map: HashMap<String, String>,
262 accept_invalid_certs: bool,
263 cache_dir: Option<PathBuf>,
264}
265
266const DEFAULT_BROKER_URL: &str = "https://api.bgpkit.com/v3/broker";
268
269fn resolve_broker_url() -> String {
272 normalize_broker_url(std::env::var("BGPKIT_BROKER_URL").ok())
273}
274
275fn normalize_broker_url(value: Option<String>) -> String {
276 match value {
277 Some(url) => {
278 let trimmed = url.trim();
279 if trimmed.is_empty() {
280 DEFAULT_BROKER_URL.to_string()
281 } else {
282 trimmed.trim_end_matches('/').to_string()
283 }
284 }
285 None => DEFAULT_BROKER_URL.to_string(),
286 }
287}
288
289impl Default for BgpkitBroker {
290 fn default() -> Self {
291 dotenvy::dotenv().ok();
292 let url = resolve_broker_url();
293
294 let collector_project_map = DEFAULT_COLLECTORS_CONFIG.clone().to_project_map();
295
296 let accept_invalid_certs = read_accept_invalid_certs_from_env();
297 let client = build_blocking_client(accept_invalid_certs);
298
299 Self {
300 broker_url: url,
301 query_params: Default::default(),
302 client,
303 collector_project_map,
304 accept_invalid_certs,
305 cache_dir: None,
306 }
307 }
308}
309
310fn read_accept_invalid_certs_from_env() -> bool {
311 match std::env::var("ONEIO_ACCEPT_INVALID_CERTS") {
312 Ok(t) => {
313 let l = t.to_lowercase();
314 l.starts_with("true") || l.starts_with("y")
315 }
316 Err(_) => false,
317 }
318}
319
320fn build_blocking_client(accept_invalid_certs: bool) -> reqwest::blocking::Client {
321 match reqwest::blocking::ClientBuilder::new()
322 .danger_accept_invalid_certs(accept_invalid_certs)
323 .user_agent(SDK_USER_AGENT)
324 .build()
325 {
326 Ok(c) => c,
327 Err(e) => {
328 panic!("Failed to build HTTP client for broker requests: {}", e);
329 }
330 }
331}
332
333#[cfg(feature = "sse")]
334pub(crate) fn build_async_client(
335 accept_invalid_certs: bool,
336) -> Result<reqwest::Client, BrokerError> {
337 reqwest::ClientBuilder::new()
338 .danger_accept_invalid_certs(accept_invalid_certs)
339 .user_agent(SDK_USER_AGENT)
340 .build()
341 .map_err(BrokerError::NetworkError)
342}
343
344impl BgpkitBroker {
345 pub fn new() -> Self {
358 Self::default()
359 }
360
361 pub fn broker_url<S: Display>(self, url: S) -> Self {
372 let broker_url = url.to_string().trim_end_matches('/').to_string();
373 Self {
374 broker_url,
375 query_params: self.query_params,
376 client: self.client,
377 collector_project_map: self.collector_project_map,
378 accept_invalid_certs: self.accept_invalid_certs,
379 cache_dir: self.cache_dir,
380 }
381 }
382
383 pub fn accept_invalid_certs(self) -> Self {
385 Self {
386 broker_url: self.broker_url,
387 query_params: self.query_params,
388 client: build_blocking_client(true),
389 collector_project_map: self.collector_project_map,
390 accept_invalid_certs: true,
391 cache_dir: self.cache_dir,
392 }
393 }
394
395 #[deprecated(since = "0.7.1", note = "Please use `accept_invalid_certs` instead.")]
397 pub fn disable_ssl_check(self) -> Self {
398 Self::accept_invalid_certs(self)
399 }
400
401 pub fn cache_dir<P: Into<PathBuf>>(mut self, path: P) -> Self {
416 let path = path.into();
417 if !path.exists() {
418 std::fs::create_dir_all(&path).expect("Failed to create cache directory");
419 }
420 self.cache_dir = Some(path);
421 self
422 }
423
424 fn cache_key(&self) -> String {
426 use sha2::{Digest, Sha256};
427
428 let params_str = format!(
429 "{}:{}:{}:{}:{}:{}:{}:{}",
430 self.broker_url,
431 self.query_params.ts_start.as_deref().unwrap_or(""),
432 self.query_params.ts_end.as_deref().unwrap_or(""),
433 self.query_params.collector_id.as_deref().unwrap_or(""),
434 self.query_params.project.as_deref().unwrap_or(""),
435 self.query_params.data_type.as_deref().unwrap_or(""),
436 self.query_params.page,
437 self.query_params.page_size
438 );
439
440 let mut hasher = Sha256::new();
441 hasher.update(params_str.as_bytes());
442 hasher
443 .finalize()
444 .iter()
445 .map(|b| format!("{:02x}", b))
446 .collect::<String>()
447 }
448
449 fn load_cache(&self) -> Option<Vec<BrokerItem>> {
451 let cache_dir = self.cache_dir.as_ref()?;
452 let cache_file = cache_dir.join(self.cache_key()).with_extension("json");
453
454 if !cache_file.exists() {
455 return None;
456 }
457
458 match std::fs::read_to_string(&cache_file) {
459 Ok(contents) => match serde_json::from_str::<Vec<BrokerItem>>(&contents) {
460 Ok(items) => {
461 log::info!("Loaded {} items from cache", items.len());
462 Some(items)
463 }
464 Err(e) => {
465 log::warn!("Failed to deserialize cache file: {}", e);
466 None
467 }
468 },
469 Err(e) => {
470 log::warn!("Failed to read cache file: {}", e);
471 None
472 }
473 }
474 }
475
476 fn save_cache(&self, items: &[BrokerItem]) {
478 let Some(cache_dir) = self.cache_dir.as_ref() else {
479 return;
480 };
481
482 let cache_file = cache_dir.join(self.cache_key()).with_extension("json");
483
484 match serde_json::to_string(items) {
485 Ok(json) => {
486 if let Err(e) = std::fs::write(&cache_file, json) {
487 log::warn!("Failed to write cache file: {}", e);
488 } else {
489 log::info!("Saved {} items to cache", items.len());
490 }
491 }
492 Err(e) => {
493 log::warn!("Failed to serialize items for cache: {}", e);
494 }
495 }
496 }
497
498 fn parse_timestamp(timestamp: &str) -> Result<DateTime<Utc>, BrokerError> {
512 let ts_str = timestamp.trim();
513
514 if let Ok(dt_with_tz) = DateTime::parse_from_rfc3339(ts_str) {
516 return Ok(dt_with_tz.with_timezone(&Utc));
517 }
518
519 if let Ok(naive_dt) = chrono::NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%dT%H:%M:%SZ") {
521 return Ok(Utc.from_utc_datetime(&naive_dt));
522 }
523
524 if let Ok(naive_dt) = chrono::NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%dT%H:%M:%S") {
526 return Ok(Utc.from_utc_datetime(&naive_dt));
527 }
528
529 if let Ok(naive_dt) = chrono::NaiveDateTime::parse_from_str(ts_str, "%Y-%m-%d %H:%M:%S") {
531 return Ok(Utc.from_utc_datetime(&naive_dt));
532 }
533
534 let date_formats = [
536 "%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d", "%Y%m%d", ];
541
542 for format in &date_formats {
543 if let Ok(date) = NaiveDate::parse_from_str(ts_str, format) {
544 if format == &"%Y%m%d" && ts_str.len() != 8 {
546 continue;
547 }
548 if let Some(naive_datetime) = date.and_hms_opt(0, 0, 0) {
550 return Ok(Utc.from_utc_datetime(&naive_datetime));
551 }
552 }
553 }
554
555 if ts_str.len() >= 9 && ts_str.len() <= 13 && ts_str.chars().all(|c| c.is_ascii_digit()) {
557 if let Ok(timestamp) = ts_str.parse::<i64>() {
558 if let Some(dt) = Utc.timestamp_opt(timestamp, 0).single() {
559 return Ok(dt);
560 }
561 }
562 }
563
564 Err(BrokerError::ConfigurationError(format!(
565 "Invalid timestamp format '{ts_str}'. Supported formats:\n\
566 - Unix timestamp: '1640995200'\n\
567 - RFC3339 with timezone: '2022-01-01T00:00:00+00:00', '2022-01-01T00:00:00Z', '2022-01-01T05:00:00-05:00'\n\
568 - RFC3339 without timezone: '2022-01-01T00:00:00' (assumes UTC)\n\
569 - Date with time: '2022-01-01 00:00:00'\n\
570 - Pure date: '2022-01-01', '2022/01/01', '2022.01.01', '20220101'"
571 )))
572 }
573
574 fn validate_configuration(&self) -> Result<QueryParams, BrokerError> {
579 let mut normalized_params = self.query_params.clone();
581
582 if normalized_params.ts_start.is_none() && normalized_params.ts_end.is_none() {
585 let now = chrono::Utc::now();
586 let thirty_days_ago = now - chrono::Duration::days(30);
587 normalized_params.ts_start =
588 Some(thirty_days_ago.format("%Y-%m-%dT%H:%M:%SZ").to_string());
589 normalized_params.ts_end = Some(now.format("%Y-%m-%dT%H:%M:%SZ").to_string());
590 }
591
592 if let Some(ts) = &self.query_params.ts_start {
593 let parsed_datetime = Self::parse_timestamp(ts)?;
594 normalized_params.ts_start =
595 Some(parsed_datetime.format("%Y-%m-%dT%H:%M:%SZ").to_string());
596 }
597
598 if let Some(ts) = &self.query_params.ts_end {
599 let parsed_datetime = Self::parse_timestamp(ts)?;
600 normalized_params.ts_end =
601 Some(parsed_datetime.format("%Y-%m-%dT%H:%M:%SZ").to_string());
602 }
603
604 if let Some(collector_str) = &self.query_params.collector_id {
606 let collectors: Vec<String> = collector_str
607 .split(',')
608 .map(|s| s.trim())
609 .filter(|s| !s.is_empty())
610 .map(|s| s.to_string())
611 .collect();
612
613 if collectors.is_empty() {
614 return Err(BrokerError::ConfigurationError(
615 "Collector ID cannot be empty".to_string(),
616 ));
617 }
618
619 let mut seen = HashSet::new();
621 let mut deduped = Vec::with_capacity(collectors.len());
622 for c in collectors {
623 if seen.insert(c.clone()) {
624 deduped.push(c);
625 }
626 }
627
628 normalized_params.collector_id = Some(deduped.join(","));
629 }
630
631 if let Some(project_str) = &self.query_params.project {
633 let project_lower = project_str.to_lowercase();
634 match project_lower.as_str() {
635 "rrc" | "riperis" | "ripe_ris" | "routeviews" | "route_views" | "rv" => {
636 }
638 _ => {
639 return Err(BrokerError::ConfigurationError(format!(
640 "Invalid project '{project_str}'. Valid projects are: 'riperis' (aliases: 'rrc', 'ripe_ris') or 'routeviews' (aliases: 'route_views', 'rv')"
641 )));
642 }
643 }
644 }
645
646 if let Some(data_type_str) = &self.query_params.data_type {
648 let data_type_lower = data_type_str.to_lowercase();
649 match data_type_lower.as_str() {
650 "rib" | "ribs" | "r" | "update" | "updates" => {
651 }
653 _ => {
654 return Err(BrokerError::ConfigurationError(format!(
655 "Invalid data type '{data_type_str}'. Valid data types are: 'rib' (aliases: 'ribs', 'r') or 'updates' (alias: 'update')"
656 )));
657 }
658 }
659 }
660
661 if self.query_params.page < 1 {
663 return Err(BrokerError::ConfigurationError(format!(
664 "Invalid page number {}. Page number must be >= 1",
665 self.query_params.page
666 )));
667 }
668
669 if !(1..=100000).contains(&self.query_params.page_size) {
671 return Err(BrokerError::ConfigurationError(format!(
672 "Invalid page size {}. Page size must be between 1 and 100000",
673 self.query_params.page_size
674 )));
675 }
676
677 Ok(normalized_params)
678 }
679
680 pub fn ts_start<S: Display>(self, ts_start: S) -> Self {
713 let mut query_params = self.query_params;
714 query_params.ts_start = Some(ts_start.to_string());
715 Self {
716 broker_url: self.broker_url,
717 query_params,
718 client: self.client,
719 collector_project_map: self.collector_project_map,
720 accept_invalid_certs: self.accept_invalid_certs,
721 cache_dir: self.cache_dir,
722 }
723 }
724
725 pub fn ts_end<S: Display>(self, ts_end: S) -> Self {
750 let mut query_params = self.query_params;
751 query_params.ts_end = Some(ts_end.to_string());
752 Self {
753 broker_url: self.broker_url,
754 client: self.client,
755 query_params,
756 collector_project_map: self.collector_project_map,
757 accept_invalid_certs: self.accept_invalid_certs,
758 cache_dir: self.cache_dir,
759 }
760 }
761
762 pub fn collector_id<S: Display>(self, collector_id: S) -> Self {
781 let mut query_params = self.query_params;
782 query_params.collector_id = Some(collector_id.to_string());
783 Self {
784 client: self.client,
785 broker_url: self.broker_url,
786 query_params,
787 collector_project_map: self.collector_project_map,
788 accept_invalid_certs: self.accept_invalid_certs,
789 cache_dir: self.cache_dir,
790 }
791 }
792
793 pub fn project<S: Display>(self, project: S) -> Self {
807 let mut query_params = self.query_params;
808 query_params.project = Some(project.to_string());
809 Self {
810 client: self.client,
811 broker_url: self.broker_url,
812 query_params,
813 collector_project_map: self.collector_project_map,
814 accept_invalid_certs: self.accept_invalid_certs,
815 cache_dir: self.cache_dir,
816 }
817 }
818
819 pub fn data_type<S: Display>(self, data_type: S) -> Self {
835 let mut query_params = self.query_params;
836 query_params.data_type = Some(data_type.to_string());
837 Self {
838 broker_url: self.broker_url,
839 client: self.client,
840 query_params,
841 collector_project_map: self.collector_project_map,
842 accept_invalid_certs: self.accept_invalid_certs,
843 cache_dir: self.cache_dir,
844 }
845 }
846
847 pub fn page(self, page: i64) -> Self {
859 let mut query_params = self.query_params;
860 query_params.page = page;
861 Self {
862 broker_url: self.broker_url,
863 client: self.client,
864 query_params,
865 collector_project_map: self.collector_project_map,
866 accept_invalid_certs: self.accept_invalid_certs,
867 cache_dir: self.cache_dir,
868 }
869 }
870
871 pub fn page_size(self, page_size: i64) -> Self {
883 let mut query_params = self.query_params;
884 query_params.page_size = page_size;
885 Self {
886 broker_url: self.broker_url,
887 client: self.client,
888 query_params,
889 collector_project_map: self.collector_project_map,
890 accept_invalid_certs: self.accept_invalid_certs,
891 cache_dir: self.cache_dir,
892 }
893 }
894
895 pub fn peers_ip(self, peer_ip: IpAddr) -> Self {
904 let mut query_params = self.query_params;
905 query_params.peers_ip = Some(peer_ip);
906 Self {
907 broker_url: self.broker_url,
908 client: self.client,
909 query_params,
910 collector_project_map: self.collector_project_map,
911 accept_invalid_certs: self.accept_invalid_certs,
912 cache_dir: self.cache_dir,
913 }
914 }
915
916 pub fn peers_asn(self, peer_asn: u32) -> Self {
925 let mut query_params = self.query_params;
926 query_params.peers_asn = Some(peer_asn);
927 Self {
928 broker_url: self.broker_url,
929 client: self.client,
930 query_params,
931 collector_project_map: self.collector_project_map,
932 accept_invalid_certs: self.accept_invalid_certs,
933 cache_dir: self.cache_dir,
934 }
935 }
936
937 pub fn peers_only_full_feed(self, peer_full_feed: bool) -> Self {
946 let mut query_params = self.query_params;
947 query_params.peers_only_full_feed = peer_full_feed;
948 Self {
949 broker_url: self.broker_url,
950 client: self.client,
951 query_params,
952 collector_project_map: self.collector_project_map,
953 accept_invalid_certs: self.accept_invalid_certs,
954 cache_dir: self.cache_dir,
955 }
956 }
957
958 pub fn turn_page(&mut self, page: i64) {
973 self.query_params.page = page;
974 }
975
976 pub fn query_single_page(&self) -> Result<Vec<BrokerItem>, BrokerError> {
986 if let Some(cached_items) = self.load_cache() {
988 return Ok(cached_items);
989 }
990
991 let validated_params = self.validate_configuration()?;
992 let url = format!("{}/search{}", self.broker_url, validated_params);
993 log::info!("sending broker query to {}", url);
994 match self.run_files_query(url.as_str()) {
995 Ok(res) => {
996 self.save_cache(&res.data);
998 Ok(res.data)
999 }
1000 Err(e) => Err(e),
1001 }
1002 }
1003
1004 pub fn query_total_count(&self) -> Result<i64, BrokerError> {
1028 let validated_params = self.validate_configuration()?;
1029 let url = format!("{}/search{}", self.broker_url, validated_params);
1030 match self.run_files_query(url.as_str()) {
1031 Ok(res) => res.total.ok_or(BrokerError::BrokerError(
1032 "count not found in response".to_string(),
1033 )),
1034 Err(e) => Err(e),
1035 }
1036 }
1037
1038 pub fn health_check(&self) -> Result<(), BrokerError> {
1047 let url = format!("{}/health", self.broker_url.trim_end_matches('/'));
1048 match self.client.get(url.as_str()).send() {
1049 Ok(response) => {
1050 if response.status() == reqwest::StatusCode::OK {
1051 Ok(())
1052 } else {
1053 Err(BrokerError::BrokerError(format!(
1054 "endpoint unhealthy {}",
1055 self.broker_url
1056 )))
1057 }
1058 }
1059 Err(_e) => Err(BrokerError::BrokerError(format!(
1060 "endpoint unhealthy {}",
1061 self.broker_url
1062 ))),
1063 }
1064 }
1065
1066 pub fn query(&self) -> Result<Vec<BrokerItem>, BrokerError> {
1085 let mut p = self.validate_configuration()?;
1086
1087 let mut items = vec![];
1088 loop {
1089 let url = format!("{}/search{}", self.broker_url, p);
1090
1091 let res_items = self.run_files_query(url.as_str())?.data;
1092
1093 let items_count = res_items.len() as i64;
1094
1095 if items_count == 0 {
1096 break;
1098 }
1099
1100 items.extend(res_items);
1101 let cur_page = p.page;
1102 p = p.page(cur_page + 1);
1103
1104 if items_count < p.page_size {
1105 break;
1107 }
1108 }
1109 Ok(items)
1110 }
1111
1112 pub fn latest(&self) -> Result<Vec<BrokerItem>, BrokerError> {
1126 let latest_query_url = format!("{}/latest", self.broker_url);
1127 let mut items = match self.client.get(latest_query_url.as_str()).send() {
1128 Ok(response) => match response.json::<CollectorLatestResult>() {
1129 Ok(result) => result.data,
1130 Err(_) => {
1131 return Err(BrokerError::BrokerError(
1132 "Error parsing response".to_string(),
1133 ));
1134 }
1135 },
1136 Err(e) => {
1137 return Err(BrokerError::BrokerError(format!(
1138 "Unable to connect to the URL ({latest_query_url}): {e}"
1139 )));
1140 }
1141 };
1142
1143 items.retain(|item| {
1144 let mut matches = true;
1145 if let Some(project) = &self.query_params.project {
1146 match project.to_lowercase().as_str() {
1147 "rrc" | "riperis" | "ripe_ris" => {
1148 matches = self
1149 .collector_project_map
1150 .get(&item.collector_id)
1151 .cloned()
1152 .unwrap_or_default()
1153 .as_str()
1154 == "riperis";
1155 }
1156 "routeviews" | "route_views" | "rv" => {
1157 matches = self
1158 .collector_project_map
1159 .get(&item.collector_id)
1160 .cloned()
1161 .unwrap_or_default()
1162 .as_str()
1163 == "routeviews";
1164 }
1165 _ => {}
1166 }
1167 }
1168
1169 if let Some(data_type) = &self.query_params.data_type {
1170 match data_type.to_lowercase().as_str() {
1171 "rib" | "ribs" | "r" if !item.is_rib() => {
1172 matches = false
1174 }
1175 "update" | "updates" if item.is_rib() => {
1176 matches = false
1178 }
1179 _ => {}
1180 }
1181 }
1182
1183 if let Some(collector_id) = &self.query_params.collector_id {
1184 let wanted: HashSet<&str> = collector_id
1185 .split(',')
1186 .map(|s| s.trim())
1187 .filter(|s| !s.is_empty())
1188 .collect();
1189
1190 if !wanted.contains(item.collector_id.as_str()) {
1191 return false;
1192 }
1193 }
1194
1195 matches
1196 });
1197
1198 Ok(items)
1199 }
1200
1201 pub fn get_peers(&self) -> Result<Vec<BrokerPeer>, BrokerError> {
1273 let mut url = format!("{}/peers", self.broker_url);
1274 let mut param_strings = vec![];
1275 if let Some(ip) = &self.query_params.peers_ip {
1276 param_strings.push(format!("ip={ip}"));
1277 }
1278 if let Some(asn) = &self.query_params.peers_asn {
1279 param_strings.push(format!("asn={asn}"));
1280 }
1281 if self.query_params.peers_only_full_feed {
1282 param_strings.push("full_feed=true".to_string());
1283 }
1284 if let Some(collector_id) = &self.query_params.collector_id {
1285 param_strings.push(format!("collector={collector_id}"));
1286 }
1287 if !param_strings.is_empty() {
1288 let param_string = param_strings.join("&");
1289 url = format!("{url}?{param_string}");
1290 }
1291
1292 let peers = match self.client.get(url.as_str()).send() {
1293 Ok(response) => match response.json::<BrokerPeersResult>() {
1294 Ok(result) => result.data,
1295 Err(_) => {
1296 return Err(BrokerError::BrokerError(
1297 "Error parsing response".to_string(),
1298 ));
1299 }
1300 },
1301 Err(e) => {
1302 return Err(BrokerError::BrokerError(format!(
1303 "Unable to connect to the URL ({url}): {e}"
1304 )));
1305 }
1306 };
1307 Ok(peers)
1308 }
1309
1310 fn run_files_query(&self, url: &str) -> Result<BrokerQueryResult, BrokerError> {
1311 log::info!("sending broker query to {}", url);
1312 match self.client.get(url).send() {
1313 Ok(res) => {
1314 let status = res.status();
1315 if !status.is_success() {
1316 let body = res.text().unwrap_or_default();
1321 let body = body.trim();
1322 let body: String = body.chars().take(512).collect();
1323 return Err(BrokerError::BrokerError(format!(
1324 "broker API returned HTTP {status}: {body}"
1325 )));
1326 }
1327 match res.json::<BrokerQueryResult>() {
1328 Ok(res) => {
1329 if let Some(e) = res.error {
1330 Err(BrokerError::BrokerError(e))
1331 } else {
1332 Ok(res)
1333 }
1334 }
1335 Err(e) => {
1336 Err(BrokerError::BrokerError(e.to_string()))
1339 }
1340 }
1341 }
1342 Err(e) => Err(BrokerError::from(e)),
1343 }
1344 }
1345}
1346
1347pub struct BrokerItemIterator {
1374 broker: BgpkitBroker,
1375 cached_items: Vec<BrokerItem>,
1376 first_run: bool,
1377}
1378
1379impl BrokerItemIterator {
1380 pub fn new(broker: BgpkitBroker) -> BrokerItemIterator {
1381 BrokerItemIterator {
1382 broker,
1383 cached_items: vec![],
1384 first_run: true,
1385 }
1386 }
1387}
1388
1389impl Iterator for BrokerItemIterator {
1390 type Item = BrokerItem;
1391
1392 fn next(&mut self) -> Option<Self::Item> {
1393 if let Some(item) = self.cached_items.pop() {
1395 return Some(item);
1396 }
1397
1398 if self.first_run {
1400 self.first_run = false;
1402 } else {
1403 self.broker.query_params.page += 1;
1405 }
1406
1407 let items = match self.broker.query_single_page() {
1409 Ok(i) => i,
1410 Err(_) => return None,
1411 };
1412
1413 if items.is_empty() {
1414 return None;
1416 } else {
1417 self.cached_items = items;
1419 self.cached_items.reverse();
1420 }
1421
1422 #[allow(clippy::unwrap_used)]
1423 Some(self.cached_items.pop().unwrap())
1424 }
1425}
1426
1427impl IntoIterator for BgpkitBroker {
1428 type Item = BrokerItem;
1429 type IntoIter = BrokerItemIterator;
1430
1431 fn into_iter(self) -> Self::IntoIter {
1432 BrokerItemIterator::new(self)
1433 }
1434}
1435
1436impl IntoIterator for &BgpkitBroker {
1437 type Item = BrokerItem;
1438 type IntoIter = BrokerItemIterator;
1439
1440 fn into_iter(self) -> Self::IntoIter {
1441 BrokerItemIterator::new(self.clone())
1442 }
1443}
1444
1445#[cfg(test)]
1446mod tests {
1447 use super::*;
1448
1449 #[test]
1452 fn test_http_error_body_is_surfaced() {
1453 use std::io::{BufRead, BufReader, Write};
1454 use std::net::TcpListener;
1455
1456 let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port");
1457 let addr = listener.local_addr().expect("local addr");
1458 let server = std::thread::spawn(move || {
1459 if let Ok((mut stream, _)) = listener.accept() {
1460 let mut reader = BufReader::new(stream.try_clone().expect("clone stream"));
1461 let mut request_line = String::new();
1462 let _ = reader.read_line(&mut request_line);
1463 let body = r#"{"error":"Failed to fetch search results","message":"All deployment instances are down"}"#;
1464 let response = format!(
1465 "HTTP/1.1 500 Internal Server Error\r\n\
1466 Content-Type: application/json\r\n\
1467 Content-Length: {}\r\n\
1468 Connection: close\r\n\r\n{}",
1469 body.len(),
1470 body
1471 );
1472 let _ = stream.write_all(response.as_bytes());
1473 let _ = stream.flush();
1474 }
1475 });
1476
1477 let broker = BgpkitBroker::new()
1478 .broker_url(format!("http://{}", addr))
1479 .ts_start("1634693400")
1480 .ts_end("1634693400");
1481 let err = broker
1482 .query_single_page()
1483 .expect_err("HTTP 500 must be reported as an error");
1484 let msg = err.to_string();
1485 assert!(msg.contains("HTTP 500"), "unexpected message: {msg}");
1486 assert!(
1487 msg.contains("All deployment instances are down"),
1488 "server error body was dropped: {msg}"
1489 );
1490
1491 let _ = server.join();
1492 }
1493
1494 #[test]
1495 fn test_query() {
1496 let broker = BgpkitBroker::new()
1497 .ts_start("1634693400")
1498 .ts_end("1634693400");
1499 let res = broker.query();
1500 assert!(&res.is_ok());
1501 let data = res.unwrap();
1502 assert!(!data.is_empty());
1503 }
1504
1505 #[test]
1506 fn test_network_error() {
1507 let broker = BgpkitBroker::new().broker_url("https://api.broker.example.com/v2");
1508 let res = broker.query();
1509 assert!(res.is_err());
1511 assert!(matches!(res.err(), Some(BrokerError::NetworkError(_))));
1512 }
1513
1514 #[test]
1515 fn test_broker_error() {
1516 let broker = BgpkitBroker::new().page(-1);
1517 let result = broker.query();
1518 assert!(result.is_err());
1519 assert!(matches!(
1520 result.err(),
1521 Some(BrokerError::ConfigurationError(_))
1522 ));
1523 }
1524
1525 #[test]
1526 fn test_query_all() {
1527 let broker = BgpkitBroker::new()
1528 .ts_start("1634693400")
1529 .ts_end("1634693400")
1530 .page_size(100);
1531 let res = broker.query();
1532 assert!(res.is_ok());
1533 assert!(res.ok().unwrap().len() >= 54);
1534 }
1535
1536 #[test]
1537 fn test_iterator() {
1538 let broker = BgpkitBroker::new()
1539 .ts_start("1634693400")
1540 .ts_end("1634693400");
1541 assert!(broker.into_iter().count() >= 54);
1542 }
1543
1544 #[test]
1545 fn test_filters() {
1546 let broker = BgpkitBroker::new()
1547 .ts_start("1634693400")
1548 .ts_end("1634693400");
1549 let items = broker.query().unwrap();
1550 assert!(items.len() >= 54);
1551
1552 let broker = BgpkitBroker::new()
1553 .ts_start("1634693400")
1554 .ts_end("1634693400")
1555 .collector_id("rrc00");
1556 let items = broker.query().unwrap();
1557 assert_eq!(items.len(), 1);
1558
1559 let broker = BgpkitBroker::new()
1560 .ts_start("1634693400")
1561 .ts_end("1634693400")
1562 .project("riperis");
1563 let items = broker.query().unwrap();
1564 assert_eq!(items.len(), 23);
1565 }
1566
1567 #[test]
1568 fn test_latest() {
1569 let broker = BgpkitBroker::new();
1570 let items = broker.latest().unwrap();
1571 assert!(items.len() >= 125);
1572
1573 let broker = BgpkitBroker::new().project("routeviews".to_string());
1574 let items = broker.latest().unwrap();
1575 assert!(!items.is_empty());
1576 assert!(items
1577 .iter()
1578 .all(|item| !item.collector_id.starts_with("rrc")));
1579
1580 let broker = BgpkitBroker::new().project("riperis".to_string());
1581 let items = broker.latest().unwrap();
1582 assert!(!items.is_empty());
1583 assert!(items
1584 .iter()
1585 .all(|item| item.collector_id.starts_with("rrc")));
1586
1587 let broker = BgpkitBroker::new().data_type("rib".to_string());
1588 let items = broker.latest().unwrap();
1589 assert!(!items.is_empty());
1590 assert!(items.iter().all(|item| item.is_rib()));
1591
1592 let broker = BgpkitBroker::new().data_type("update".to_string());
1593 let items = broker.latest().unwrap();
1594 assert!(!items.is_empty());
1595 assert!(items.iter().all(|item| !item.is_rib()));
1596
1597 let broker = BgpkitBroker::new().collector_id("rrc00".to_string());
1598 let items = broker.latest().unwrap();
1599 assert!(!items.is_empty());
1600 assert!(items
1601 .iter()
1602 .all(|item| item.collector_id.as_str() == "rrc00"));
1603 assert_eq!(items.len(), 2);
1604 }
1605
1606 #[test]
1607 fn test_latest_no_ssl() {
1608 let broker = BgpkitBroker::new().accept_invalid_certs();
1609 let items = broker.latest().unwrap();
1610 assert!(items.len() >= 125);
1611 }
1612
1613 #[test]
1614 fn test_health_check() {
1615 let broker = BgpkitBroker::new();
1616 let res = broker.health_check();
1617 assert!(res.is_ok());
1618 }
1619
1620 #[test]
1621 fn test_peers() {
1622 let broker = BgpkitBroker::new();
1623 let all_peers = broker.get_peers().unwrap();
1624 assert!(!all_peers.is_empty());
1625 let first_peer = all_peers.first().unwrap();
1626 let first_ip = first_peer.ip;
1627 let first_asn = first_peer.asn;
1628
1629 let broker = BgpkitBroker::new().peers_ip(first_ip);
1630 let peers = broker.get_peers().unwrap();
1631 assert!(!peers.is_empty());
1632
1633 let broker = BgpkitBroker::new().peers_asn(first_asn);
1634 let peers = broker.get_peers().unwrap();
1635 assert!(!peers.is_empty());
1636
1637 let broker = BgpkitBroker::new().peers_only_full_feed(true);
1638 let full_feed_peers = broker.get_peers().unwrap();
1639 assert!(!full_feed_peers.is_empty());
1640 assert!(full_feed_peers.len() < all_peers.len());
1641
1642 let broker = BgpkitBroker::new().collector_id("rrc00");
1643 let rrc_peers = broker.get_peers().unwrap();
1644 assert!(!rrc_peers.is_empty());
1645 assert!(rrc_peers.iter().all(|peer| peer.collector == "rrc00"));
1646
1647 let broker = BgpkitBroker::new().collector_id("rrc00,route-views2");
1648 let rrc_rv_peers = broker.get_peers().unwrap();
1649 assert!(!rrc_rv_peers.is_empty());
1650 assert!(rrc_rv_peers
1651 .iter()
1652 .any(|peer| peer.collector == "rrc00" || peer.collector == "route-views2"));
1653
1654 assert!(rrc_rv_peers.len() > rrc_peers.len());
1655 }
1656
1657 #[test]
1658 fn test_timestamp_parsing_unix() {
1659 let broker = BgpkitBroker::new();
1660
1661 let result = broker.clone().ts_start("1640995200");
1663 assert_eq!(result.query_params.ts_start, Some("1640995200".to_string()));
1665
1666 let result = broker.clone().ts_end("1640995200");
1667 assert_eq!(result.query_params.ts_end, Some("1640995200".to_string()));
1668 }
1669
1670 #[test]
1671 fn test_timestamp_parsing_rfc3339() {
1672 let broker = BgpkitBroker::new();
1673
1674 let result = broker.clone().ts_start("2022-01-01T00:00:00Z");
1676 assert_eq!(
1677 result.query_params.ts_start,
1678 Some("2022-01-01T00:00:00Z".to_string())
1679 );
1680
1681 let result = broker.clone().ts_start("2022-01-01T12:30:45");
1683 assert_eq!(
1684 result.query_params.ts_start,
1685 Some("2022-01-01T12:30:45".to_string())
1686 );
1687
1688 let result = broker.clone().ts_end("2022-01-01 12:30:45");
1690 assert_eq!(
1691 result.query_params.ts_end,
1692 Some("2022-01-01 12:30:45".to_string())
1693 );
1694 }
1695
1696 #[test]
1697 fn test_timestamp_parsing_pure_dates() {
1698 let broker = BgpkitBroker::new();
1699
1700 let result = broker.clone().ts_start("2022-01-01");
1702 assert_eq!(result.query_params.ts_start, Some("2022-01-01".to_string()));
1703
1704 let result = broker.clone().ts_start("2022/01/01");
1706 assert_eq!(result.query_params.ts_start, Some("2022/01/01".to_string()));
1707
1708 let result = broker.clone().ts_end("2022.01.01");
1710 assert_eq!(result.query_params.ts_end, Some("2022.01.01".to_string()));
1711
1712 let result = broker.clone().ts_end("20220101");
1714 assert_eq!(result.query_params.ts_end, Some("20220101".to_string()));
1715 }
1716
1717 #[test]
1718 fn test_timestamp_parsing_whitespace() {
1719 let broker = BgpkitBroker::new();
1720
1721 let result = broker.clone().ts_start(" 2022-01-01 ");
1723 assert_eq!(
1724 result.query_params.ts_start,
1725 Some(" 2022-01-01 ".to_string())
1726 );
1727
1728 let result = broker.clone().ts_end("\t1640995200\n");
1729 assert_eq!(
1730 result.query_params.ts_end,
1731 Some("\t1640995200\n".to_string())
1732 );
1733 }
1734
1735 #[test]
1736 fn test_timestamp_parsing_errors() {
1737 let broker = BgpkitBroker::new();
1738
1739 let broker_with_invalid = broker.clone().ts_start("invalid-timestamp");
1741 let result = broker_with_invalid.query();
1742 assert!(result.is_err());
1743 assert!(matches!(
1744 result.err(),
1745 Some(BrokerError::ConfigurationError(_))
1746 ));
1747
1748 let broker_with_invalid = broker.clone().ts_end("2022-13-01");
1750 let result = broker_with_invalid.query();
1751 assert!(result.is_err());
1752 assert!(matches!(
1753 result.err(),
1754 Some(BrokerError::ConfigurationError(_))
1755 ));
1756
1757 let broker_with_invalid = broker.clone().ts_start("20221301");
1759 let result = broker_with_invalid.query();
1760 assert!(result.is_err());
1761 assert!(matches!(
1762 result.err(),
1763 Some(BrokerError::ConfigurationError(_))
1764 ));
1765
1766 let broker_with_invalid = broker.clone().ts_start("2022-01");
1768 let result = broker_with_invalid.query();
1769 assert!(result.is_err());
1770 assert!(matches!(
1771 result.err(),
1772 Some(BrokerError::ConfigurationError(_))
1773 ));
1774 }
1775
1776 #[test]
1777 fn test_parse_timestamp_direct() {
1778 use chrono::{NaiveDate, NaiveDateTime};
1779
1780 let expected_unix = Utc.timestamp_opt(1640995200, 0).single().unwrap();
1784 assert_eq!(
1785 BgpkitBroker::parse_timestamp("1640995200").unwrap(),
1786 expected_unix
1787 );
1788
1789 let expected_rfc3339_z = Utc.from_utc_datetime(
1791 &NaiveDateTime::parse_from_str("2022-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S").unwrap(),
1792 );
1793 assert_eq!(
1794 BgpkitBroker::parse_timestamp("2022-01-01T00:00:00Z").unwrap(),
1795 expected_rfc3339_z
1796 );
1797
1798 let expected_rfc3339_no_z = Utc.from_utc_datetime(
1799 &NaiveDateTime::parse_from_str("2022-01-01T12:30:45", "%Y-%m-%dT%H:%M:%S").unwrap(),
1800 );
1801 assert_eq!(
1802 BgpkitBroker::parse_timestamp("2022-01-01T12:30:45").unwrap(),
1803 expected_rfc3339_no_z
1804 );
1805
1806 let expected_space_format = Utc.from_utc_datetime(
1807 &NaiveDateTime::parse_from_str("2022-01-01 12:30:45", "%Y-%m-%d %H:%M:%S").unwrap(),
1808 );
1809 assert_eq!(
1810 BgpkitBroker::parse_timestamp("2022-01-01 12:30:45").unwrap(),
1811 expected_space_format
1812 );
1813
1814 let expected_date = Utc.from_utc_datetime(
1816 &NaiveDate::from_ymd_opt(2022, 1, 1)
1817 .unwrap()
1818 .and_hms_opt(0, 0, 0)
1819 .unwrap(),
1820 );
1821 assert_eq!(
1822 BgpkitBroker::parse_timestamp("2022-01-01").unwrap(),
1823 expected_date
1824 );
1825 assert_eq!(
1826 BgpkitBroker::parse_timestamp("2022/01/01").unwrap(),
1827 expected_date
1828 );
1829 assert_eq!(
1830 BgpkitBroker::parse_timestamp("2022.01.01").unwrap(),
1831 expected_date
1832 );
1833 assert_eq!(
1834 BgpkitBroker::parse_timestamp("20220101").unwrap(),
1835 expected_date
1836 );
1837
1838 let result_plus_tz = BgpkitBroker::parse_timestamp("2022-01-01T00:00:00+00:00").unwrap();
1840 assert_eq!(result_plus_tz, expected_date);
1841 println!("✓ +00:00 timezone format works");
1842
1843 let result_minus_tz = BgpkitBroker::parse_timestamp("2022-01-01T05:00:00-05:00").unwrap();
1845 let expected_10am = Utc.with_ymd_and_hms(2022, 1, 1, 10, 0, 0).unwrap();
1846 assert_eq!(result_minus_tz, expected_10am);
1847 println!("✓ -05:00 timezone format works (05:00-05:00 = 10:00Z)");
1848
1849 assert!(BgpkitBroker::parse_timestamp("invalid").is_err());
1851 assert!(BgpkitBroker::parse_timestamp("2022-13-01").is_err());
1852 assert!(BgpkitBroker::parse_timestamp("2022-01").is_err());
1853 }
1854
1855 #[test]
1856 fn test_collector_id_validation() {
1857 let broker = BgpkitBroker::new();
1858
1859 let broker_valid = broker.clone().collector_id("rrc00");
1861 let result = broker_valid.validate_configuration();
1862 assert!(result.is_ok());
1863
1864 let broker_valid = broker.clone().collector_id("rrc00,route-views2");
1866 let result = broker_valid.validate_configuration();
1867 assert!(result.is_ok());
1868
1869 let broker_unknown = broker.clone().collector_id("brand-new-collector");
1871 let result = broker_unknown.validate_configuration();
1872 assert!(result.is_ok());
1873
1874 let broker_mixed = broker.clone().collector_id("rrc00,brand-new-collector");
1876 let result = broker_mixed.validate_configuration();
1877 assert!(result.is_ok());
1878
1879 let broker_empty = broker.clone().collector_id(", , ,");
1881 let result = broker_empty.validate_configuration();
1882 assert!(result.is_err());
1883 assert!(matches!(
1884 result.err(),
1885 Some(BrokerError::ConfigurationError(_))
1886 ));
1887 }
1888
1889 #[test]
1890 fn test_project_validation() {
1891 let broker = BgpkitBroker::new();
1892
1893 let broker_valid = broker.clone().project("riperis");
1895 let result = broker_valid.validate_configuration();
1896 assert!(result.is_ok());
1897
1898 let broker_valid = broker.clone().project("routeviews");
1899 let result = broker_valid.validate_configuration();
1900 assert!(result.is_ok());
1901
1902 let broker_valid = broker.clone().project("rrc");
1904 let result = broker_valid.validate_configuration();
1905 assert!(result.is_ok());
1906
1907 let broker_valid = broker.clone().project("rv");
1908 let result = broker_valid.validate_configuration();
1909 assert!(result.is_ok());
1910
1911 let broker_invalid = broker.clone().project("invalid-project");
1913 let result = broker_invalid.validate_configuration();
1914 assert!(result.is_err());
1915 assert!(matches!(
1916 result.err(),
1917 Some(BrokerError::ConfigurationError(_))
1918 ));
1919 }
1920
1921 #[test]
1922 fn test_data_type_validation() {
1923 let broker = BgpkitBroker::new();
1924
1925 let broker_valid = broker.clone().data_type("rib");
1927 let result = broker_valid.validate_configuration();
1928 assert!(result.is_ok());
1929
1930 let broker_valid = broker.clone().data_type("updates");
1931 let result = broker_valid.validate_configuration();
1932 assert!(result.is_ok());
1933
1934 let broker_valid = broker.clone().data_type("ribs");
1936 let result = broker_valid.validate_configuration();
1937 assert!(result.is_ok());
1938
1939 let broker_valid = broker.clone().data_type("update");
1940 let result = broker_valid.validate_configuration();
1941 assert!(result.is_ok());
1942
1943 let broker_invalid = broker.clone().data_type("invalid-type");
1945 let result = broker_invalid.validate_configuration();
1946 assert!(result.is_err());
1947 assert!(matches!(
1948 result.err(),
1949 Some(BrokerError::ConfigurationError(_))
1950 ));
1951 }
1952
1953 #[test]
1954 fn test_page_validation() {
1955 let broker = BgpkitBroker::new();
1956
1957 let broker_valid = broker.clone().page(1);
1959 let result = broker_valid.validate_configuration();
1960 assert!(result.is_ok());
1961
1962 let broker_valid = broker.clone().page(100);
1963 let result = broker_valid.validate_configuration();
1964 assert!(result.is_ok());
1965
1966 let broker_invalid = broker.clone().page(0);
1968 let result = broker_invalid.validate_configuration();
1969 assert!(result.is_err());
1970 assert!(matches!(
1971 result.err(),
1972 Some(BrokerError::ConfigurationError(_))
1973 ));
1974 }
1975
1976 #[test]
1977 fn test_page_size_validation() {
1978 let broker = BgpkitBroker::new();
1979
1980 let broker_valid = broker.clone().page_size(1);
1982 let result = broker_valid.validate_configuration();
1983 assert!(result.is_ok());
1984
1985 let broker_valid = broker.clone().page_size(100);
1986 let result = broker_valid.validate_configuration();
1987 assert!(result.is_ok());
1988
1989 let broker_valid = broker.clone().page_size(100000);
1990 let result = broker_valid.validate_configuration();
1991 assert!(result.is_ok());
1992
1993 let broker_invalid = broker.clone().page_size(0);
1995 let result = broker_invalid.validate_configuration();
1996 assert!(result.is_err());
1997 assert!(matches!(
1998 result.err(),
1999 Some(BrokerError::ConfigurationError(_))
2000 ));
2001
2002 let broker_invalid = broker.clone().page_size(100001);
2003 let result = broker_invalid.validate_configuration();
2004 assert!(result.is_err());
2005 assert!(matches!(
2006 result.err(),
2007 Some(BrokerError::ConfigurationError(_))
2008 ));
2009 }
2010
2011 #[test]
2012 fn test_method_chaining() {
2013 let broker = BgpkitBroker::new()
2014 .ts_start("1634693400")
2015 .ts_end("1634693400")
2016 .collector_id("rrc00")
2017 .project("riperis")
2018 .data_type("rib")
2019 .page(1)
2020 .page_size(10);
2021
2022 assert_eq!(broker.query_params.ts_start, Some("1634693400".to_string()));
2024 assert_eq!(broker.query_params.ts_end, Some("1634693400".to_string()));
2025 assert_eq!(broker.query_params.collector_id, Some("rrc00".to_string()));
2026 assert_eq!(broker.query_params.project, Some("riperis".to_string()));
2027 assert_eq!(broker.query_params.data_type, Some("rib".to_string()));
2028 assert_eq!(broker.query_params.page, 1);
2029 assert_eq!(broker.query_params.page_size, 10);
2030 }
2031
2032 #[test]
2033 fn test_broker_url_falls_back_to_default() {
2034 assert_eq!(normalize_broker_url(None), DEFAULT_BROKER_URL);
2035 assert_eq!(
2036 normalize_broker_url(Some(String::new())),
2037 DEFAULT_BROKER_URL
2038 );
2039 assert_eq!(
2040 normalize_broker_url(Some(" ".to_string())),
2041 DEFAULT_BROKER_URL
2042 );
2043 }
2044
2045 #[test]
2046 fn test_broker_url_is_normalized() {
2047 assert_eq!(
2048 normalize_broker_url(Some(" https://broker.example.com/v2/ ".to_string())),
2049 "https://broker.example.com/v2"
2050 );
2051 assert_eq!(
2052 normalize_broker_url(Some("https://broker.example.com/v2////".to_string())),
2053 "https://broker.example.com/v2"
2054 );
2055 }
2056}