flyby 0.1.1

A high-performance Rust framework for composable data-ingestion pipelines.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

//! [`NetworkSource`]: the batch-poll trait for network backends.
//!
//! Every network backend (AF_XDP, DPDK, simulator, pcap replay) implements
//! this trait. The pipeline drives it by calling [`poll_batch`] in a loop
//! and forwarding each [`RawBatch`] to the decoder stage.
//!
//! ## Relationship to [`crate::core::Source`]
//!
//! [`crate::core::Source`] returns one byte slice per poll. `NetworkSource`
//! extends the model with a batch API, which is essential for network
//! ingest where polling in bursts amortises system-call overhead.
//!
//! A future pipeline revision will drive `NetworkSource` directly.
//! For now, backends also implement `crate::core::Source` as a compatibility
//! shim (returning the first packet of each batch).
//!
//! [`poll_batch`]: NetworkSource::poll_batch

use crate::core::{Lifecycle, Result};

use crate::net::batch::RawBatch;

/// What the source should do when the downstream pipeline cannot keep up.
///
/// The default policy for new backends is [`DropNewest`][Self::DropNewest].
/// Production deployments must choose a policy explicitly and configure
/// their metrics to surface drop events — FlyBy never silently drops.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum BackpressurePolicy {
    /// Discard the incoming packet when the pipeline is full.
    /// Keeps the ring flowing; latency of older packets is unaffected.
    #[default]
    DropNewest,
    /// Evict the oldest unread packet to make room for the new one.
    /// Preserves recency at the cost of losing older data.
    DropOldest,
    /// Spin until the pipeline can accept the packet.
    /// Zero packet loss; may stall the source under sustained load.
    Block,
    /// Forward overflow packets to a configured overflow sink.
    /// Requires an overflow sink to be registered with the pipeline.
    Overflow,
}

/// A network packet source that produces raw bytes in batches.
///
/// All network backends — simulated, AF_XDP, DPDK, pcap replay — implement
/// this trait. The pipeline calls [`poll_batch`][Self::poll_batch] in a
/// tight loop. Back-pressure is reported via [`backpressure_policy`][Self::backpressure_policy]
/// and tracked via [`RawBatch::record_drop`].
pub trait NetworkSource: Lifecycle {
    /// Poll up to `batch.capacity()` packets into `batch`.
    ///
    /// The implementation must call [`batch.reset`][RawBatch::reset] before
    /// filling new packets. Returns the number of packets received (which
    /// equals `batch.len()` after the call).
    ///
    /// `Ok(0)` means no packets were available (the source is idle).
    /// `Err(_)` means a non-recoverable backend failure.
    fn poll_batch(&mut self, batch: &mut RawBatch) -> Result<usize>;

    /// The backpressure policy this source applies when the pipeline is full.
    fn backpressure_policy(&self) -> BackpressurePolicy {
        BackpressurePolicy::DropNewest
    }

    /// Human-readable name of this backend, used in logs and metrics.
    fn backend_name(&self) -> &'static str;
}