flyby 0.1.1

A high-performance Rust framework for composable data-ingestion pipelines.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
//! Configuration types for network backends.
//!
//! Each backend has its own config struct. All are deliberately plain data
//! (no methods, no validation inside the struct) so they can be constructed
//! from TOML, environment variables, or code equally easily.
//!
//! ## Example (TOML)
//!
//! ```toml
//! [source]
//! kind = "af_xdp"
//! interface = "eth1"
//! queue_id = 0
//! mode = "copy"
//! poll_budget = 64
//!
//! [source.xdp]
//! program = "redirect"
//! filter_udp_port = 9000
//! attach_mode = "native"
//!
//! [source.umem]
//! frame_size = 2048
//! frame_count = 4096
//! ```

/// AF_XDP copy/zero-copy operating mode.
///
/// Always set this explicitly. When `Auto` is selected the driver tries
/// zero-copy and falls back to copy when unavailable. Silent downgrade is
/// not acceptable: the active mode **must** be logged and emitted as a
/// metric (ADR-0004).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum XdpMode {
    /// The kernel copies each packet from the NIC ring into UMEM.
    /// Works on any NIC with a kernel driver. Preferred for initial
    /// development and testing.
    #[default]
    Copy,
    /// The NIC DMA's packets directly into UMEM. Requires a compatible
    /// NIC driver and correct queue setup. Benchmark before claiming
    /// any performance advantage over copy mode.
    ZeroCopy,
    /// Try zero-copy; fall back to copy with mandatory log + metric.
    Auto,
}

impl XdpMode {
    /// Returns `true` if this mode may perform zero-copy transfer.
    pub fn may_zero_copy(self) -> bool {
        matches!(self, XdpMode::ZeroCopy | XdpMode::Auto)
    }
}

/// XDP/eBPF program configuration.
///
/// The XDP program runs in the kernel and is responsible for redirecting
/// packets to the AF_XDP socket. It must stay minimal: packet filtering
/// only. Business logic belongs in userspace.
///
/// # Requirements
///
/// - Linux kernel ≥ 5.4 (≥ 5.10 recommended for zero-copy stability).
/// - `CAP_SYS_ADMIN` or `CAP_BPF` for program loading.
/// - `CAP_NET_ADMIN` for XDP attachment.
///
/// # Warning
///
/// Zero-copy mode additionally requires a NIC driver that exports AF_XDP
/// support (`ethtool --show-features eth0 | grep xdp`). Docker Desktop
/// on macOS and most GitHub-hosted CI runners do **not** provide this.
#[derive(Debug, Clone)]
pub struct XdpConfig {
    /// Which XDP program to load. `"redirect"` is the built-in
    /// pass-through-and-redirect program.
    pub program: String,
    /// If non-zero, only redirect UDP packets on this destination port.
    pub filter_udp_port: u16,
    /// XDP attachment mode: `"native"`, `"generic"`, or `"offload"`.
    ///
    /// `"native"` is preferred when the driver supports it.
    /// `"generic"` (SKB mode) works on any driver but has higher overhead.
    pub attach_mode: String,
}

impl Default for XdpConfig {
    fn default() -> Self {
        Self {
            program: "redirect".into(),
            filter_udp_port: 0,
            attach_mode: "native".into(),
        }
    }
}

/// UMEM (userspace memory) configuration for the AF_XDP backend.
///
/// UMEM is the memory region shared between the kernel and the AF_XDP
/// socket. It holds packet frame buffers. It is a **separate memory
/// domain** from the FlyBy shared-memory sink — do not confuse them.
///
/// True end-to-end zero-copy (UMEM → shared-memory sink without a copy)
/// is a separate and harder problem that is not claimed in v0.1.
#[derive(Debug, Clone)]
pub struct UmemConfig {
    /// Size in bytes of each UMEM frame. Must be a power of two.
    /// Typical values: 2048, 4096.
    pub frame_size: usize,
    /// Number of frames in the UMEM region. Must be a power of two.
    pub frame_count: usize,
}

impl Default for UmemConfig {
    fn default() -> Self {
        Self {
            frame_size: 2048,
            frame_count: 4096,
        }
    }
}

/// Full configuration for the AF_XDP source backend.
///
/// # Hardware requirements
///
/// - Linux host (not Docker Desktop on macOS).
/// - Kernel ≥ 5.4 for copy mode; ≥ 5.10 for zero-copy.
/// - `CAP_SYS_ADMIN` or `CAP_BPF` + `CAP_NET_ADMIN`.
/// - NIC driver with AF_XDP support for zero-copy mode.
///
/// # CI limitations
///
/// GitHub-hosted runners cannot run AF_XDP. Use the simulator or a
/// self-hosted Linux runner with a compatible NIC for hardware tests.
#[derive(Debug, Clone)]
pub struct AfXdpConfig {
    /// Network interface name (e.g. `"eth1"`, `"ens3"`).
    pub interface: String,
    /// NIC queue index to bind. Pinning to a specific queue is strongly
    /// recommended to avoid cross-queue coordination.
    pub queue_id: u32,
    /// Copy or zero-copy mode.
    pub mode: XdpMode,
    /// Maximum packets to pull from the RX ring per poll call.
    pub poll_budget: usize,
    /// XDP/eBPF program settings.
    pub xdp: XdpConfig,
    /// UMEM layout settings.
    pub umem: UmemConfig,
}

impl Default for AfXdpConfig {
    fn default() -> Self {
        Self {
            interface: "eth0".into(),
            queue_id: 0,
            mode: XdpMode::Copy,
            poll_budget: 64,
            xdp: XdpConfig::default(),
            umem: UmemConfig::default(),
        }
    }
}

/// Configuration for the in-process simulated network source.
///
/// Useful for developing parsers, placement logic, and sinks without
/// real hardware. The simulator generates Ethernet/IP/UDP shaped packets.
///
/// Call [`SimNetConfig::validate`] before use (also invoked by
/// [`crate::net::sim::SimulatedNetSource::try_new`] and `init`).
#[derive(Debug, Clone)]
pub struct SimNetConfig {
    /// Payload bytes appended after the UDP header.
    /// Default: 8 bytes (a u64 sequence number, big-endian).
    pub payload_size: usize,
    /// Packets to attempt per [`poll_batch`][crate::net::source::NetworkSource::poll_batch]
    /// call. Must be > 0. When larger than the batch capacity, excess is
    /// counted as drops.
    pub batch_size: usize,
    /// Fraction of polls that return zero packets (simulate idle NIC).
    /// Must be in `[0.0, 1.0)`.
    pub idle_rate: f32,
    /// Fraction of packets to deliberately drop (simulates NIC drops).
    /// Must be in `[0.0, 1.0)`.
    pub drop_rate: f32,
    /// UDP destination port written into simulated packet headers.
    pub udp_dst_port: u16,
}

impl Default for SimNetConfig {
    fn default() -> Self {
        Self {
            payload_size: 8,
            batch_size: 32,
            idle_rate: 0.0,
            drop_rate: 0.0,
            udp_dst_port: 9000,
        }
    }
}

impl SimNetConfig {
    /// Validate configuration constraints.
    pub fn validate(&self) -> crate::core::Result<()> {
        if self.batch_size == 0 {
            return Err(crate::core::Error::config("batch_size must be > 0"));
        }
        if !(0.0..1.0).contains(&self.idle_rate) {
            return Err(crate::core::Error::config(
                "idle_rate must be in [0.0, 1.0)",
            ));
        }
        if !(0.0..1.0).contains(&self.drop_rate) {
            return Err(crate::core::Error::config(
                "drop_rate must be in [0.0, 1.0)",
            ));
        }
        Ok(())
    }
}

impl UmemConfig {
    /// Validate UMEM geometry.
    pub fn validate(&self) -> crate::core::Result<()> {
        if self.frame_size == 0 || !self.frame_size.is_power_of_two() {
            return Err(crate::core::Error::config(
                "umem frame_size must be a non-zero power of two",
            ));
        }
        if self.frame_count == 0 || !self.frame_count.is_power_of_two() {
            return Err(crate::core::Error::config(
                "umem frame_count must be a non-zero power of two",
            ));
        }
        Ok(())
    }
}

impl AfXdpConfig {
    /// Validate AF_XDP configuration.
    pub fn validate(&self) -> crate::core::Result<()> {
        if self.interface.is_empty() {
            return Err(crate::core::Error::config("interface must not be empty"));
        }
        if self.poll_budget == 0 {
            return Err(crate::core::Error::config("poll_budget must be > 0"));
        }
        self.umem.validate()?;
        Ok(())
    }
}

impl DpdkConfig {
    /// Validate DPDK configuration.
    pub fn validate(&self) -> crate::core::Result<()> {
        if self.pci_addr.is_empty() {
            return Err(crate::core::Error::config("pci_addr must not be empty"));
        }
        if self.burst_size == 0 {
            return Err(crate::core::Error::config("burst_size must be > 0"));
        }
        Ok(())
    }
}

/// Configuration for the DPDK source backend (design placeholder).
///
/// # Requirements
///
/// - External DPDK installation (≥ 22.11 recommended).
/// - Hugepages configured (`/sys/kernel/mm/hugepages/`).
/// - NIC bound to a VFIO or UIO driver.
/// - EAL arguments (core mask, memory channels, device PCI address).
///
/// # Status
///
/// DPDK is deferred after AF_XDP (see ADR-002). This struct defines the
/// intended configuration surface; the binding is a future deliverable.
#[derive(Debug, Clone)]
pub struct DpdkConfig {
    /// PCI address of the NIC (e.g. `"0000:00:1f.6"`).
    pub pci_addr: String,
    /// EAL core mask (e.g. `"0x3"` for cores 0 and 1).
    pub core_mask: String,
    /// Number of hugepages to pre-allocate.
    pub hugepage_count: usize,
    /// RX queue index to bind.
    pub rx_queue_id: u16,
    /// Maximum packets per burst receive call.
    pub burst_size: u16,
}

impl Default for DpdkConfig {
    fn default() -> Self {
        Self {
            pci_addr: String::new(),
            core_mask: "0x1".into(),
            hugepage_count: 512,
            rx_queue_id: 0,
            burst_size: 32,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn xdp_mode_and_defaults() {
        assert!(!XdpMode::Copy.may_zero_copy());
        assert!(XdpMode::ZeroCopy.may_zero_copy());
        assert!(XdpMode::Auto.may_zero_copy());
        assert_eq!(XdpConfig::default().program, "redirect");
        assert_eq!(UmemConfig::default().frame_size, 2048);
        assert_eq!(AfXdpConfig::default().interface, "eth0");
        assert_eq!(SimNetConfig::default().udp_dst_port, 9000);
    }

    #[test]
    fn validates_umem_and_af_xdp() {
        UmemConfig::default().validate().unwrap();
        assert!(
            UmemConfig {
                frame_size: 3,
                frame_count: 4
            }
            .validate()
            .is_err()
        );
        assert!(
            UmemConfig {
                frame_size: 2048,
                frame_count: 0
            }
            .validate()
            .is_err()
        );

        AfXdpConfig::default().validate().unwrap();
        assert!(
            AfXdpConfig {
                interface: String::new(),
                ..AfXdpConfig::default()
            }
            .validate()
            .is_err()
        );
        assert!(
            AfXdpConfig {
                poll_budget: 0,
                ..AfXdpConfig::default()
            }
            .validate()
            .is_err()
        );
    }

    #[test]
    fn validates_sim_and_dpdk() {
        SimNetConfig::default().validate().unwrap();
        assert!(
            SimNetConfig {
                batch_size: 0,
                ..SimNetConfig::default()
            }
            .validate()
            .is_err()
        );
        assert!(
            SimNetConfig {
                idle_rate: 1.0,
                ..SimNetConfig::default()
            }
            .validate()
            .is_err()
        );
        assert!(
            SimNetConfig {
                drop_rate: -0.1,
                ..SimNetConfig::default()
            }
            .validate()
            .is_err()
        );

        assert!(DpdkConfig::default().validate().is_err());
        DpdkConfig {
            pci_addr: "0000:00:00.0".into(),
            ..DpdkConfig::default()
        }
        .validate()
        .unwrap();
        assert!(
            DpdkConfig {
                pci_addr: "x".into(),
                burst_size: 0,
                ..DpdkConfig::default()
            }
            .validate()
            .is_err()
        );
    }
}