use std::collections::{HashMap, VecDeque};
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::Duration;
use ahash::RandomState;
use bytes::Bytes;
use flowscope::tracker::FlowEvents;
use flowscope::{
EndReason, FlowEvent, FlowExtractor, FlowSide, FlowTracker, FlowTrackerConfig, PacketView,
Timestamp,
};
use futures_core::Stream;
use crate::async_adapters::async_reassembler::{AsyncReassembler, AsyncReassemblerFactory};
use crate::async_adapters::tokio_adapter::AsyncCapture;
use crate::dedup::Dedup;
use crate::error::Error;
use crate::traits::PacketSource;
pub struct NoReassembler;
pub struct AsyncReassemblerSlot<K, F>
where
K: Eq + std::hash::Hash + Clone + Send + 'static,
F: AsyncReassemblerFactory<K>,
{
factory: F,
instances: HashMap<(K, FlowSide), F::Reassembler, RandomState>,
pending_payloads: VecDeque<(K, FlowSide, u32, Bytes)>,
pending_future: Option<Pin<Box<dyn Future<Output = ()> + Send + 'static>>>,
}
pub struct FlowStream<S, E, U = (), R = NoReassembler>
where
S: PacketSource + std::os::unix::io::AsRawFd,
E: FlowExtractor,
U: Send + 'static,
{
cap: AsyncCapture<S>,
tracker: FlowTracker<E, U>,
pending: VecDeque<FlowEvent<E::Key>>,
sweep: tokio::time::Interval,
reassembler: R,
dedup: Option<Dedup>,
monotonic_ts: Option<Timestamp>,
#[cfg(feature = "pcap")]
tap: Option<crate::pcap_tap::PcapTap>,
}
impl<S, E> FlowStream<S, E, (), NoReassembler>
where
S: PacketSource + std::os::unix::io::AsRawFd,
E: FlowExtractor,
{
pub(crate) fn new(cap: AsyncCapture<S>, extractor: E) -> Self {
let tracker = FlowTracker::new(extractor);
let sweep_interval = tracker.config().sweep_interval;
Self {
cap,
tracker,
pending: VecDeque::new(),
sweep: tokio::time::interval(sweep_interval),
reassembler: NoReassembler,
dedup: None,
monotonic_ts: None,
#[cfg(feature = "pcap")]
tap: None,
}
}
pub fn with_state<U, F>(self, init: F) -> FlowStream<S, E, U, NoReassembler>
where
U: Send + 'static,
F: FnMut(&E::Key) -> U + Send + Sync + 'static,
{
let config = self.tracker.config().clone();
let extractor = self.tracker.into_extractor();
FlowStream {
cap: self.cap,
tracker: FlowTracker::with_config_and_state(extractor, config, init),
pending: VecDeque::new(),
sweep: self.sweep,
reassembler: NoReassembler,
dedup: self.dedup,
monotonic_ts: self.monotonic_ts,
#[cfg(feature = "pcap")]
tap: self.tap,
}
}
}
impl<S, E, U> FlowStream<S, E, U, NoReassembler>
where
S: PacketSource + std::os::unix::io::AsRawFd,
E: FlowExtractor,
U: Send + 'static,
{
pub fn with_async_reassembler<F>(
self,
factory: F,
) -> FlowStream<S, E, U, AsyncReassemblerSlot<E::Key, F>>
where
F: AsyncReassemblerFactory<E::Key>,
{
FlowStream {
cap: self.cap,
tracker: self.tracker,
pending: self.pending,
sweep: self.sweep,
reassembler: AsyncReassemblerSlot {
factory,
instances: HashMap::with_hasher(RandomState::new()),
pending_payloads: VecDeque::new(),
pending_future: None,
},
dedup: self.dedup,
monotonic_ts: self.monotonic_ts,
#[cfg(feature = "pcap")]
tap: self.tap,
}
}
}
impl<S, E> FlowStream<S, E, (), NoReassembler>
where
S: PacketSource + std::os::unix::io::AsRawFd,
E: FlowExtractor,
E::Key: Eq + std::hash::Hash + Clone + Send + 'static,
{
pub fn session_stream<F>(
self,
factory: F,
) -> crate::async_adapters::session_stream::SessionStream<S, E, F>
where
F: flowscope::SessionParserFactory<E::Key>,
{
crate::async_adapters::session_stream::SessionStream::from_tracker(
self.cap,
self.tracker,
factory,
self.dedup,
self.monotonic_ts,
#[cfg(feature = "pcap")]
self.tap,
)
}
pub fn datagram_stream<F>(
self,
factory: F,
) -> crate::async_adapters::datagram_stream::DatagramStream<S, E, F>
where
F: flowscope::DatagramParserFactory<E::Key>,
{
crate::async_adapters::datagram_stream::DatagramStream::from_tracker(
self.cap,
self.tracker,
factory,
self.dedup,
self.monotonic_ts,
#[cfg(feature = "pcap")]
self.tap,
)
}
}
impl<S, E, U, R> FlowStream<S, E, U, R>
where
S: PacketSource + std::os::unix::io::AsRawFd,
E: FlowExtractor,
U: Send + 'static,
{
pub fn with_config(mut self, config: FlowTrackerConfig) -> Self {
let new_interval = config.sweep_interval;
self.tracker.set_config(config);
self.sweep = tokio::time::interval(new_interval);
self
}
pub fn with_dedup(mut self, dedup: Dedup) -> Self {
self.dedup = Some(dedup);
self
}
pub fn dedup(&self) -> Option<&Dedup> {
self.dedup.as_ref()
}
pub fn dedup_mut(&mut self) -> Option<&mut Dedup> {
self.dedup.as_mut()
}
pub fn tracker(&self) -> &FlowTracker<E, U> {
&self.tracker
}
pub fn tracker_mut(&mut self) -> &mut FlowTracker<E, U> {
&mut self.tracker
}
pub fn with_idle_timeout_fn<F>(mut self, f: F) -> Self
where
F: Fn(&E::Key, Option<flowscope::L4Proto>) -> Option<Duration> + Send + Sync + 'static,
{
self.tracker.set_idle_timeout_fn(f);
self
}
pub fn with_monotonic_timestamps(mut self, enable: bool) -> Self {
self.monotonic_ts = if enable {
Some(Timestamp::default())
} else {
None
};
self
}
pub fn snapshot_flow_stats(
&self,
) -> impl Iterator<Item = (&E::Key, &flowscope::FlowStats)> + '_ {
self.tracker.iter_active().map(|af| (af.key, af.stats))
}
pub fn tracker_stats(&self) -> &flowscope::FlowTrackerStats {
self.tracker.stats()
}
pub fn active_flows(&self) -> usize {
self.tracker.flows().count()
}
#[cfg(feature = "pcap")]
pub fn with_pcap_tap<W>(self, writer: crate::pcap::CaptureWriter<W>) -> Self
where
W: std::io::Write + Send + 'static,
{
self.with_pcap_tap_policy(writer, crate::pcap_tap::TapErrorPolicy::default())
}
#[cfg(feature = "pcap")]
pub fn with_pcap_tap_policy<W>(
mut self,
writer: crate::pcap::CaptureWriter<W>,
policy: crate::pcap_tap::TapErrorPolicy,
) -> Self
where
W: std::io::Write + Send + 'static,
{
self.tap = Some(crate::pcap_tap::PcapTap::new(writer, policy));
self
}
#[cfg(feature = "pcap")]
pub fn with_pcap_tap_snaplen(mut self, snaplen: u32) -> Self {
if let Some(tap) = self.tap.as_mut() {
tap.set_snaplen(snaplen);
}
self
}
}
impl<S, E, U> Stream for FlowStream<S, E, U, NoReassembler>
where
S: PacketSource + std::os::unix::io::AsRawFd + Unpin,
E: FlowExtractor + Unpin,
E::Key: Clone + Unpin,
U: Send + 'static + Unpin,
{
type Item = Result<FlowEvent<E::Key>, Error>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
loop {
if let Some(evt) = this.pending.pop_front() {
return Poll::Ready(Some(Ok(evt)));
}
if this.sweep.poll_tick(cx).is_ready() {
let now = clamp_now(current_timestamp(), &mut this.monotonic_ts);
for ev in this.tracker.sweep(now) {
this.pending.push_back(ev);
}
if let Some(evt) = this.pending.pop_front() {
return Poll::Ready(Some(Ok(evt)));
}
}
let mut guard = match this.cap.poll_read_ready_mut(cx) {
Poll::Ready(Ok(g)) => g,
Poll::Ready(Err(e)) => return Poll::Ready(Some(Err(Error::Io(e)))),
Poll::Pending => return Poll::Pending,
};
let got_batch = {
let inner = guard.get_inner_mut();
if let Some(batch) = inner.next_batch() {
#[cfg(feature = "pcap")]
let mut tap_error: Option<Error> = None;
for pkt in &batch {
if let Some(d) = this.dedup.as_mut()
&& !d.keep(&pkt)
{
continue;
}
#[cfg(feature = "pcap")]
if let Some(tap) = this.tap.as_mut()
&& let Some(err) = tap.write_or_handle(&pkt)
{
tap_error = Some(err);
break;
}
let view = clamp_view(pkt.view(), &mut this.monotonic_ts);
let evts: FlowEvents<E::Key> = this.tracker.track(view);
for ev in evts {
this.pending.push_back(ev);
}
}
drop(batch);
#[cfg(feature = "pcap")]
if let Some(err) = tap_error {
return Poll::Ready(Some(Err(err)));
}
true
} else {
false
}
};
if !got_batch {
guard.clear_ready();
}
}
}
}
pub(crate) fn clamp_view<'a>(
view: PacketView<'a>,
state: &mut Option<Timestamp>,
) -> PacketView<'a> {
let Some(last) = state.as_mut() else {
return view;
};
*last = (*last).max(view.timestamp);
PacketView::new(view.frame, *last)
}
pub(crate) fn clamp_now(now: Timestamp, state: &mut Option<Timestamp>) -> Timestamp {
let Some(last) = state.as_mut() else {
return now;
};
*last = (*last).max(now);
*last
}
impl<S, E, U, F> Stream for FlowStream<S, E, U, AsyncReassemblerSlot<E::Key, F>>
where
S: PacketSource + std::os::unix::io::AsRawFd + Unpin,
E: FlowExtractor + Unpin,
E::Key: Clone + Unpin,
U: Send + 'static + Unpin,
F: AsyncReassemblerFactory<E::Key> + Unpin,
F::Reassembler: Unpin,
{
type Item = Result<FlowEvent<E::Key>, Error>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
loop {
if let Some(fut) = this.reassembler.pending_future.as_mut() {
match fut.as_mut().poll(cx) {
Poll::Ready(()) => {
this.reassembler.pending_future = None;
}
Poll::Pending => return Poll::Pending,
}
}
if let Some((key, side, seq, payload)) = this.reassembler.pending_payloads.pop_front() {
let r = this
.reassembler
.instances
.entry((key.clone(), side))
.or_insert_with(|| this.reassembler.factory.new_reassembler(&key, side));
let fut = r.segment(seq, payload);
this.reassembler.pending_future = Some(fut);
continue;
}
if let Some(evt) = this.pending.pop_front() {
if let FlowEvent::Ended { key, reason, .. } = &evt {
let reason_copy = *reason;
let key_copy = key.clone();
let mut found_fut = None;
for side in [FlowSide::Initiator, FlowSide::Responder] {
if let Some(mut r) =
this.reassembler.instances.remove(&(key_copy.clone(), side))
{
let fut = match reason_copy {
EndReason::Fin | EndReason::IdleTimeout => r.fin(),
EndReason::Rst
| EndReason::Evicted
| EndReason::BufferOverflow
| EndReason::ParseError => r.rst(),
_ => r.rst(),
};
drop(r);
found_fut = Some(fut);
break;
}
}
if let Some(fut) = found_fut {
this.pending.push_front(evt);
this.reassembler.pending_future = Some(fut);
continue;
}
}
return Poll::Ready(Some(Ok(evt)));
}
if this.sweep.poll_tick(cx).is_ready() {
let now = clamp_now(current_timestamp(), &mut this.monotonic_ts);
for ev in this.tracker.sweep(now) {
this.pending.push_back(ev);
}
if !this.pending.is_empty() {
continue;
}
}
let mut guard = match this.cap.poll_read_ready_mut(cx) {
Poll::Ready(Ok(g)) => g,
Poll::Ready(Err(e)) => return Poll::Ready(Some(Err(Error::Io(e)))),
Poll::Pending => return Poll::Pending,
};
let got_batch = {
let inner = guard.get_inner_mut();
if let Some(batch) = inner.next_batch() {
#[cfg(feature = "pcap")]
let mut tap_error: Option<Error> = None;
for pkt in &batch {
if let Some(d) = this.dedup.as_mut()
&& !d.keep(&pkt)
{
continue;
}
#[cfg(feature = "pcap")]
if let Some(tap) = this.tap.as_mut()
&& let Some(err) = tap.write_or_handle(&pkt)
{
tap_error = Some(err);
break;
}
let view = clamp_view(pkt.view(), &mut this.monotonic_ts);
let payloads = &mut this.reassembler.pending_payloads;
let evts: FlowEvents<E::Key> =
this.tracker
.track_with_payload(view, |key, side, seq, payload| {
payloads.push_back((
key.clone(),
side,
seq,
Bytes::copy_from_slice(payload),
));
});
for ev in evts {
this.pending.push_back(ev);
}
}
drop(batch);
#[cfg(feature = "pcap")]
if let Some(err) = tap_error {
return Poll::Ready(Some(Err(err)));
}
true
} else {
false
}
};
if !got_batch {
guard.clear_ready();
}
}
}
}
fn current_timestamp() -> Timestamp {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or(Duration::ZERO);
Timestamp::new(now.as_secs() as u32, now.subsec_nanos())
}
impl<S> AsyncCapture<S>
where
S: PacketSource + std::os::unix::io::AsRawFd,
{
pub fn flow_stream<E>(self, extractor: E) -> FlowStream<S, E, (), NoReassembler>
where
E: FlowExtractor,
{
FlowStream::new(self, extractor)
}
}
use crate::async_adapters::stream_capture::{Sealed, StreamCapture};
impl<S, E, U, R> Sealed for FlowStream<S, E, U, R>
where
S: PacketSource + std::os::unix::io::AsRawFd,
E: FlowExtractor,
U: Send + 'static,
{
}
impl<S, E, U, R> StreamCapture for FlowStream<S, E, U, R>
where
S: PacketSource + std::os::unix::io::AsRawFd,
E: FlowExtractor,
U: Send + 'static,
{
type Source = S;
fn capture(&self) -> &AsyncCapture<S> {
&self.cap
}
fn dedup(&self) -> Option<&Dedup> {
self.dedup.as_ref()
}
fn dedup_mut(&mut self) -> Option<&mut Dedup> {
self.dedup.as_mut()
}
}
#[cfg(test)]
mod monotonic_tests {
use super::*;
#[test]
fn clamp_view_passthrough_when_off() {
let mut state: Option<Timestamp> = None;
let frame = [0u8; 4];
let ts = Timestamp::new(100, 0);
let v = PacketView::new(&frame, ts);
let out = clamp_view(v, &mut state);
assert_eq!(out.timestamp, ts);
assert!(state.is_none());
}
#[test]
fn clamp_view_advances_running_max() {
let mut state: Option<Timestamp> = Some(Timestamp::default());
let frame = [0u8; 4];
let t1 = Timestamp::new(100, 0);
let t2 = Timestamp::new(50, 0); let t3 = Timestamp::new(200, 0);
let v1 = clamp_view(PacketView::new(&frame, t1), &mut state);
assert_eq!(v1.timestamp, t1);
assert_eq!(state, Some(t1));
let v2 = clamp_view(PacketView::new(&frame, t2), &mut state);
assert_eq!(v2.timestamp, t1, "step-back clamps to running max");
assert_eq!(state, Some(t1));
let v3 = clamp_view(PacketView::new(&frame, t3), &mut state);
assert_eq!(v3.timestamp, t3, "step-forward advances running max");
assert_eq!(state, Some(t3));
}
#[test]
fn clamp_now_passthrough_when_off() {
let mut state: Option<Timestamp> = None;
let ts = Timestamp::new(42, 0);
assert_eq!(clamp_now(ts, &mut state), ts);
assert!(state.is_none());
}
#[test]
fn clamp_now_clamps_to_running_max() {
let mut state: Option<Timestamp> = Some(Timestamp::new(100, 0));
let clamped = clamp_now(Timestamp::new(50, 0), &mut state);
assert_eq!(clamped, Timestamp::new(100, 0));
let advanced = clamp_now(Timestamp::new(200, 0), &mut state);
assert_eq!(advanced, Timestamp::new(200, 0));
assert_eq!(state, Some(Timestamp::new(200, 0)));
}
}