monocle 1.6.1

A commandline application to search, parse, and process BGP information in public sources.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
//! Watch command: stream live BGP messages from RIPE RIS Live.
//!
//! Filter semantics match `monocle parse` where the dimensions overlap. The
//! subscription pushes `host`/`prefix`/`peer`/`require` down to RIS Live to
//! reduce traffic, but all element predicates also run client-side with
//! parser semantics (RIS selects whole UPDATE messages; elements expand per
//! prefix). The stream can be recorded to an MRT updates file for offline
//! replay with `monocle parse`.

use std::io::Write;
use std::path::PathBuf;

use anyhow::{anyhow, Result};
use bgpkit_parser::encoder::MrtUpdatesEncoder;
use bgpkit_parser::RisLiveClientMessage;
use clap::Args;
use futures_util::{SinkExt, StreamExt};
use std::future::Future;
use tokio_tungstenite::connect_async;
use tokio_tungstenite::tungstenite::Message;

use monocle::lens::watch::{
    parse_live_frame, LiveFrame, WatchFilters, RECONNECT_BACKOFF, RIS_LIVE_URL,
};
use monocle::utils::{OutputFormat, TimestampFormat};

use super::elem_format::{format_elem, get_header, parse_fields};

/// Arguments for the Watch command
#[derive(Args)]
pub(crate) struct WatchArgs {
    /// RRC host to subscribe to (e.g. rrc00). All active RRCs if omitted.
    #[clap(long, short = 'H', value_name = "HOST")]
    pub host: Option<String>,

    /// Drink from the full unfiltered stream (~5k msgs/s). Required only
    /// when no filter is given at all; filters without a server-side scope
    /// (host/prefix/peer) still receive the full feed but are applied
    /// client-side.
    #[clap(long)]
    pub firehose: bool,

    /// Disable automatic reconnection on abnormal disconnects
    #[clap(long)]
    pub no_reconnect: bool,

    /// Record the filtered stream to an MRT updates file for offline replay
    /// (e.g. `monocle parse out.mrt.bz2`)
    #[clap(long, short = 'M', value_name = "PATH")]
    pub record: Option<PathBuf>,

    /// Pretty-print JSON output
    #[clap(long)]
    pub pretty: bool,

    /// Comma-separated list of fields to output
    #[clap(long, short = 'f', value_name = "FIELDS")]
    pub fields: Option<String>,

    /// Timestamp output format (unix or rfc3339)
    #[clap(long, value_enum, default_value = "unix")]
    pub time_format: TimestampFormat,

    /// Live-stream filters (pushed down to RIS Live where supported)
    #[clap(flatten)]
    pub filters: WatchFilters,
}

pub fn run(mut args: WatchArgs, output_format: OutputFormat) {
    // main() resets SIGPIPE to SIG_DFL so finite commands terminate on broken
    // pipes; for watch that would kill the process at the kernel write and
    // bypass recorder finalization. Restore the ignored disposition so writes
    // return EPIPE, which the stream loop already handles cleanly.
    #[cfg(unix)]
    {
        unsafe {
            libc::signal(libc::SIGPIPE, libc::SIG_IGN);
        }
    }

    // The tree enables two rustls CryptoProviders (aws-lc-rs via oneio, ring
    // via reqwest); pick ring once so TLS setup cannot fail ambiguously.
    install_crypto_provider();

    // Table output cannot stream (format_elem returns None for it); reject
    // before touching any file or network. --pretty only upgrades compact
    // JSON, so it does not make Table streamable either.
    if output_format == OutputFormat::Table {
        eprintln!(
            "watch: --format table is not supported for an unbounded stream; \
             use the default PSV, --json, or --format json-line"
        );
        std::process::exit(2);
    }

    // Validate filters and parse fields before opening the record file, so
    // an invalid invocation cannot truncate an existing recording.
    let client_filters = match args.filters.compile_client_filters() {
        Ok(f) => f,
        Err(e) => {
            eprintln!("watch: invalid filter: {e}");
            std::process::exit(2);
        }
    };
    let fields = match parse_fields(&args.fields, false) {
        Ok(f) => f,
        Err(e) => {
            eprintln!("watch: {e}");
            std::process::exit(2);
        }
    };

    // Firehose guard: measured against the live feed, the full stream is
    // ~5.2k messages/s and a steady ~28 MB RSS, so client-side filtering is
    // viable. Any filter dimension is therefore allowed without --firehose;
    // only a completely bare invocation (which most users hit by accident)
    // still requires the explicit opt-in. Server-side scope (--host/--prefix/
    // --peer-ip) remains worthwhile to cut bandwidth and RIPE-side load.
    let plan = match args.filters.to_subscription_plan(args.host.as_deref()) {
        Ok(p) => p,
        Err(e) => {
            eprintln!("watch: {e}");
            std::process::exit(2);
        }
    };
    if !args.firehose && args.filters.is_empty() && args.host.is_none() {
        eprintln!(
            "watch: no filter given; pass at least one filter (e.g. --origin-asn, --prefix, \
             --peer-asn, or --host to also cut server-side traffic) or pass --firehose to \
             drink the full stream (~5k msgs/s)"
        );
        std::process::exit(2);
    }

    // Create the runtime first: if it fails, nothing has been opened or
    // truncated yet. Then open the record writer before entering the async
    // runtime, because oneio wraps reqwest::blocking, whose internal tokio
    // runtime must not be created or dropped from within an async context.
    let rt = match tokio::runtime::Runtime::new() {
        Ok(rt) => rt,
        Err(e) => {
            eprintln!("Failed to create async runtime: {e}");
            std::process::exit(1);
        }
    };

    let recorder = match args.record.take() {
        Some(path) => match MrtRecorder::new(path) {
            Ok(r) => Some(r),
            Err(e) => {
                eprintln!("watch: cannot open record file: {e}");
                std::process::exit(1);
            }
        },
        None => None,
    };
    let mut recorder = recorder;
    let mut stats = WatchStats::default();
    let result = rt.block_on(run_async(
        args,
        plan,
        &mut recorder,
        &mut stats,
        client_filters,
        fields,
        output_format,
    ));
    // Finalize and verify the recording from a sync context: the compression
    // footer drops here, and replay verification goes through oneio, whose
    // reqwest::blocking client must not run inside the async runtime.
    if let Some(rec) = recorder.as_mut() {
        let finish_res = rec.finish().and_then(|()| rec.verify_replayable());
        match finish_res {
            Ok(()) => {
                eprintln!(
                    "recorded {} elements to {:?} (count-verified)",
                    stats.recorded, rec.path
                );
            }
            Err(e) => {
                eprintln!("watch: recording verification failed: {e}");
                if result.is_ok() {
                    std::process::exit(1);
                }
            }
        }
    }
    if let Err(e) = result {
        eprintln!("watch: {e}");
        std::process::exit(1);
    }
}

fn install_crypto_provider() {
    use std::sync::Once;
    static INSTALL: Once = Once::new();
    INSTALL.call_once(|| {
        let provider = rustls::crypto::ring::default_provider();
        if provider.install_default().is_err() {
            // Another provider was already installed process-wide; that one is used.
        }
    });
}

async fn run_async(
    args: WatchArgs,
    plan: monocle::lens::watch::SubscriptionPlan,
    recorder: &mut Option<MrtRecorder>,
    stats: &mut WatchStats,
    client_filters: Vec<bgpkit_parser::parser::filter::Filter>,
    fields: Vec<&'static str>,
    output_format: OutputFormat,
) -> Result<()> {
    let WatchArgs {
        no_reconnect,
        pretty,
        time_format,
        filters,
        ..
    } = args;

    // Request raw BGP bytes and a subscription acknowledgement: raw parsing
    // preserves all path attributes, and the ack distinguishes an accepted
    // subscription from a silently rejected one.
    let subscriptions: Vec<_> = plan
        .subscriptions
        .into_iter()
        .map(|s| s.include_raw(true).acknowledge(true))
        .collect();

    eprintln!("source: RIPE RIS Live ({RIS_LIVE_URL})");
    if let Some(h) = &plan.report.host {
        eprintln!("  host: {h}");
    }
    if !plan.report.prefixes.is_empty() {
        eprintln!(
            "  server-side prefixes: {}",
            plan.report.prefixes.join(", ")
        );
    }
    if !plan.report.peers.is_empty() {
        eprintln!("  server-side peers: {}", plan.report.peers.join(", "));
    }
    if let Some(req) = &plan.report.require {
        eprintln!("  server-side require: {req}");
    }
    if !client_filters.is_empty() {
        eprintln!(
            "  client-side filters: {} (server-side pushdown only reduces traffic)",
            client_filters.len()
        );
    }
    eprintln!("  live vantage is RIS collectors only, not global visibility");
    eprintln!("press Ctrl-C to stop");

    // Match parse: --pretty only upgrades compact JSON to pretty JSON.
    let out_format = if pretty && output_format == OutputFormat::Json {
        OutputFormat::JsonPretty
    } else {
        output_format
    };
    // Write the header through Write with BrokenPipe handling: println!
    // panics on a broken pipe (e.g. `watch --format markdown | head`).
    {
        let stdout = std::io::stdout();
        let mut lock = stdout.lock();
        if let Some(h) = get_header(out_format, &fields) {
            if let Err(e) = writeln!(lock, "{h}") {
                if e.kind() == std::io::ErrorKind::BrokenPipe {
                    // The reader is already gone (e.g. `| head` exited before
                    // any element): nothing was streamed or recorded. Sync
                    // cleanup in run() finalizes and verifies the recording.
                    return Ok(());
                }
                return Err(anyhow!("stdout write failed: {e}"));
            }
        }
        if let Err(e) = lock.flush() {
            if e.kind() == std::io::ErrorKind::BrokenPipe {
                return Ok(());
            }
            return Err(anyhow!("stdout flush failed: {e}"));
        }
    }

    let mut running = true;
    // One signal future for the whole session: once created, Tokio keeps
    // SIGINT registered process-wide, so Ctrl-C must be awaited during
    // connect, subscribe send, and backoff too, not only while reading.
    let mut sig = std::pin::pin!(tokio::signal::ctrl_c());

    // Errors inside the session set `fatal` and break out to the recorder
    // finalization below, so accepted elements are never lost from the
    // recorder buffer on an error path.
    let mut fatal: Option<anyhow::Error> = None;

    'session: loop {
        let (ws_stream, _) = tokio::select! {
            c = connect_async(RIS_LIVE_URL) => match c {
                Ok(c) => c,
                Err(e) => {
                    if running && !no_reconnect {
                        eprintln!("connection failed ({e}); retrying in {RECONNECT_BACKOFF:?}");
                        if fut_select_sig_or_sleep(&mut sig).await {
                            break;
                        }
                        continue 'session;
                    }
                    fatal = Some(anyhow!("connection failed: {e}"));
                    break;
                }
            },
            _ = &mut sig => break,
        };
        if !running {
            break;
        }

        eprintln!("connected");
        let (mut write, mut read) = ws_stream.split();

        let mut subscribed_ok = false;
        for sub in &subscriptions {
            let sub_msg = sub.to_json_string();
            let send = write.send(Message::Text(sub_msg.clone().into()));
            tokio::select! {
                res = send => {
                    if let Err(e) = res {
                        if running && !no_reconnect {
                            eprintln!("subscribe send failed ({e}); reconnecting");
                            // Restart the whole connection, not just the
                            // subscription loop: the socket is unusable.
                            if fut_select_sig_or_sleep(&mut sig).await {
                                running = false;
                                break;
                            }
                            continue 'session;
                        }
                        fatal = Some(anyhow!("subscribe send failed: {e}"));
                        break;
                    }
                    eprintln!("subscribed: {sub_msg}");
                }
                _ = &mut sig => {
                    running = false;
                    break;
                }
            }
        }
        if !running || fatal.is_some() {
            break;
        }

        let mut stdout = std::io::stdout();

        while running {
            let msg = tokio::select! {
                m = read.next() => match m {
                    Some(Ok(m)) => m,
                    Some(Err(e)) => {
                        eprintln!("stream error: {e}");
                        // Without reconnection an interrupted stream is a
                        // failure: keep it fatal so scripts see a nonzero exit.
                        if no_reconnect {
                            fatal = Some(anyhow!("stream error: {e}"));
                        }
                        break;
                    }
                    None => {
                        eprintln!("stream closed by server");
                        if no_reconnect {
                            fatal = Some(anyhow!("stream closed by server"));
                        }
                        break;
                    }
                },
                _ = &mut sig => {
                    running = false;
                    break;
                }
            };

            if msg.is_empty() {
                continue;
            }
            let Ok(text) = msg.to_text() else {
                continue;
            };

            stats.messages += 1;
            let frame = match parse_live_frame(text) {
                Ok(f) => f,
                Err(e) => {
                    fatal = Some(e);
                    break;
                }
            };
            match frame {
                LiveFrame::SubscribeOk => {
                    if !subscribed_ok {
                        eprintln!("subscription acknowledged by server");
                        subscribed_ok = true;
                    }
                }
                LiveFrame::Error(err_text) => {
                    fatal = Some(anyhow!("RIS Live rejected the subscription: {err_text}"));
                    break;
                }
                LiveFrame::Other => {}
                LiveFrame::Data(live) => {
                    if !subscribed_ok {
                        // Data before an ack still means the subscription works.
                        subscribed_ok = true;
                    }
                    for elem in live.elems {
                        if !filters.matches(&elem, &client_filters) {
                            continue;
                        }
                        stats.elems += 1;
                        if let Some(rec) = recorder.as_mut() {
                            if let Err(e) = rec.process(&elem) {
                                fatal = Some(e);
                                break;
                            }
                            stats.recorded += 1;
                        }
                        if let Some(line) = format_elem(
                            &elem,
                            out_format,
                            &fields,
                            live.host.as_deref(),
                            time_format,
                        ) {
                            if let Err(e) = writeln!(stdout, "{line}") {
                                if e.kind() == std::io::ErrorKind::BrokenPipe {
                                    // e.g. `| head`: stop cleanly.
                                    running = false;
                                } else {
                                    fatal = Some(anyhow!("stdout write failed: {e}"));
                                }
                                break;
                            }
                        }
                    }
                    if fatal.is_some() {
                        break;
                    }
                }
            }
        }

        // Flush recorder before deciding on reconnect.
        if let Some(rec) = recorder.as_mut() {
            if let Err(e) = rec.flush() {
                fatal = Some(e);
            }
        }

        eprintln!(
            "disconnected after {} messages; total elements: {}",
            stats.messages, stats.elems
        );

        if fatal.is_some() || !running || no_reconnect {
            break;
        }
        eprintln!("reconnecting in {RECONNECT_BACKOFF:?}");
        // A completed ctrl_c future must not be polled again; break the
        // session loop here instead of relying on the `running` check.
        if fut_select_sig_or_sleep(&mut sig).await {
            break;
        }
    }

    match fatal {
        Some(e) => Err(e),
        None => Ok(()),
    }
}

/// Await Ctrl-C (returns true) or the reconnect backoff (returns false).
///
/// Wraps the one-shot `ctrl_c` future so that once it has completed it is
/// never polled again (re-polling a completed future panics); callers must
/// treat `true` as terminal.
async fn fut_select_sig_or_sleep<F: Future<Output = std::io::Result<()>> + Unpin>(
    sig: &mut F,
) -> bool {
    let slept = tokio::time::sleep(RECONNECT_BACKOFF);
    tokio::pin!(slept);
    tokio::select! {
        _ = sig => true,
        _ = &mut slept => false,
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use bgpkit_parser::BgpkitParser;

    fn synthetic_elem(i: u32) -> bgpkit_parser::BgpElem {
        bgpkit_parser::BgpElem {
            peer_ip: "192.0.2.1".parse().unwrap(),
            peer_asn: bgpkit_parser::models::Asn::new_32bit(64512),
            prefix: "203.0.113.0/24".parse().unwrap(),
            timestamp: 1_700_000_000.0 + f64::from(i),
            elem_type: bgpkit_parser::models::ElemType::ANNOUNCE,
            ..Default::default()
        }
    }

    #[test]
    fn recorder_flushes_at_boundary_and_finalizes_partial_batch() {
        for count in [499u32, 500, 501] {
            let dir = tempfile::tempdir().unwrap();
            let path = dir.path().join(format!("rec-{count}.mrt"));
            let mut rec = MrtRecorder::new(path.clone()).unwrap();
            for i in 0..count {
                let mut elem = synthetic_elem(i);
                elem.timestamp = 1_700_000_000.0 + f64::from(i);
                rec.process(&elem).unwrap();
            }
            rec.finish().unwrap();

            // The finished file must replay to exactly the elements written.
            let parser = BgpkitParser::new(path.to_str().unwrap()).unwrap();
            let replayed = parser.into_iter().count();
            assert_eq!(replayed, count as usize, "count {count} round trip");
        }
    }

    #[test]
    fn recorder_finish_without_elements_writes_valid_empty_stream() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("empty.mrt");
        let mut rec = MrtRecorder::new(path.clone()).unwrap();
        rec.finish().unwrap();
        // File exists and is valid (possibly zero-record) MRT.
        assert!(path.exists());
    }
}

#[derive(Default)]
struct WatchStats {
    messages: u64,
    elems: u64,
    recorded: u64,
}

/// Incremental MRT updates recorder.
///
/// `MrtUpdatesEncoder` accumulates elements in memory and exports once, which
/// would make an unbounded live stream grow without bound. The recorder
/// instead flushes the encoder to the output file periodically: each flush
/// writes a self-contained batch of BGP4MP messages, and the concatenation of
/// batches is itself a valid updates MRT stream. Write failures abort the
/// command: the recording would silently lose batches otherwise.
struct MrtRecorder {
    path: PathBuf,
    encoder: MrtUpdatesEncoder,
    writer: Option<Box<dyn Write>>,
    count: u64,
}

impl MrtRecorder {
    const FLUSH_INTERVAL: u64 = 500;

    fn new(path: PathBuf) -> Result<Self> {
        let path_str = path
            .to_str()
            .ok_or_else(|| anyhow!("record path is not valid UTF-8"))?;
        let writer =
            oneio::get_writer(path_str).map_err(|e| anyhow!("cannot open record file: {e}"))?;
        Ok(Self {
            path,
            encoder: MrtUpdatesEncoder::new(),
            writer: Some(writer),
            count: 0,
        })
    }

    fn process(&mut self, elem: &bgpkit_parser::BgpElem) -> Result<()> {
        self.encoder.process_elem(elem);
        self.count += 1;
        if self.count.is_multiple_of(Self::FLUSH_INTERVAL) {
            self.flush()?;
        }
        Ok(())
    }

    fn flush(&mut self) -> Result<()> {
        let bytes = self
            .encoder
            .export_bytes()
            .map_err(|e| anyhow!("MRT encode failed: {e}"))?;
        if !bytes.is_empty() {
            if let Some(w) = self.writer.as_mut() {
                w.write_all(&bytes)
                    .map_err(|e| anyhow!("record write failed: {e}"))?;
            }
        }
        self.encoder.reset();
        Ok(())
    }

    fn finish(&mut self) -> Result<()> {
        self.flush()?;
        if let Some(w) = self.writer.as_mut() {
            w.flush().map_err(|e| anyhow!("record flush failed: {e}"))?;
        }
        // Compressed writers (e.g. bzip2) write their stream footer on drop,
        // and a drop-time encoder failure cannot propagate through
        // `Box<dyn Write>`. Drop the writer explicitly, then verify the
        // recording is replayable so silent corruption becomes a detected
        // error instead of a successful exit with a corrupt file.
        // Explicit drop: the compression footer is written here. A footer
        // failure cannot propagate through `Box<dyn Write>`; callers verify
        // replayability from a sync context (`verify_replayable`).
        drop(self.writer.take());
        Ok(())
    }

    /// Verify the finished recording replays to exactly the recorded count.
    ///
    /// Count-level only: the MRT encoder rebuilds messages from the
    /// `BgpElem` projection, and attributes outside that model (e.g.
    /// ORIGINATOR_ID, and ATOMIC_AGGREGATE in parser 0.20) do not survive
    /// the round trip, so per-attribute equality cannot be claimed.
    /// Must run OUTSIDE the async runtime: parsing goes through oneio, whose
    /// reqwest::blocking client creates its own tokio runtime.
    fn verify_replayable(&self) -> Result<()> {
        let path_str = self
            .path
            .to_str()
            .ok_or_else(|| anyhow!("record path is not valid UTF-8"))?;
        let parser = bgpkit_parser::BgpkitParser::new(path_str)
            .map_err(|e| anyhow!("recorded file is not parseable: {e}"))?;
        let replayed = parser.into_iter().count();
        if replayed != self.count as usize {
            return Err(anyhow!(
                "recorded file verification failed: wrote {} elements but replayed {replayed}",
                self.count
            ));
        }
        Ok(())
    }
}