netflow_parser/lib.rs
1//! # netflow_parser
2//!
3//! A Netflow Parser library for Cisco V5, V7, V9, and IPFIX written in Rust. Supports chaining of multiple versions in the same stream.
4//!
5//! > **⚠️ Breaking Changes in v0.7.0:** The Template TTL API has been simplified to only support time-based expiration.
6//! > Packet-based and combined TTL modes have been removed. See the [RELEASES.md](https://github.com/mikemiles-dev/netflow_parser/blob/main/RELEASES.md)
7//! > for the full migration guide.
8//!
9//! ## Quick Start
10//!
11//! ### Using the Builder Pattern (Recommended)
12//!
13//! ```rust
14//! use netflow_parser::NetflowParser;
15//! use netflow_parser::variable_versions::ttl::TtlConfig;
16//! use std::time::Duration;
17//!
18//! // Create a parser with custom configuration
19//! let mut parser = NetflowParser::builder()
20//! .with_cache_size(2000)
21//! .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
22//! .build()
23//! .expect("Failed to build parser");
24//!
25//! // Parse packets
26//! let buffer = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7];
27//! let packets = parser.parse_bytes(&buffer);
28//! ```
29//!
30//! ### Using Default Configuration
31//!
32//! ```rust
33//! use netflow_parser::{NetflowParser, NetflowPacket};
34//!
35//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
36//! match NetflowParser::default().parse_bytes(&v5_packet).first() {
37//! Some(NetflowPacket::V5(v5)) => assert_eq!(v5.header.version, 5),
38//! Some(NetflowPacket::Error(e)) => println!("{:?}", e),
39//! _ => (),
40//! }
41//! ```
42//!
43//! ## Want Serialization such as JSON?
44//! Structures fully support serialization. Below is an example using the serde_json macro:
45//! ```rust
46//! use serde_json::json;
47//! use netflow_parser::NetflowParser;
48//!
49//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
50//! println!("{}", json!(NetflowParser::default().parse_bytes(&v5_packet)).to_string());
51//! ```
52//!
53//! ```json
54//! [
55//! {
56//! "V5": {
57//! "header": {
58//! "count": 1,
59//! "engine_id": 7,
60//! "engine_type": 6,
61//! "flow_sequence": 33752069,
62//! "sampling_interval": 2057,
63//! "sys_up_time": { "nanos": 672000000, "secs": 50332 },
64//! "unix_nsecs": 134807553,
65//! "unix_secs": 83887623,
66//! "version": 5
67//! },
68//! "sets": [
69//! {
70//! "d_octets": 66051,
71//! "d_pkts": 101124105,
72//! "dst_addr": "4.5.6.7",
73//! "dst_as": 515,
74//! "dst_mask": 5,
75//! "dst_port": 1029,
76//! "first": { "nanos": 87000000, "secs": 67438 },
77//! "input": 515,
78//! "last": { "nanos": 553000000, "secs": 134807 },
79//! "next_hop": "8.9.0.1",
80//! "output": 1029,
81//! "pad1": 6,
82//! "pad2": 1543,
83//! "protocol_number": 8,
84//! "protocol_type": "Egp",
85//! "src_addr": "0.1.2.3",
86//! "src_as": 1,
87//! "src_mask": 4,
88//! "src_port": 515,
89//! "tcp_flags": 7,
90//! "tos": 9
91//! }
92//! ]
93//! }
94//! }
95//! ]
96//! ```
97//!
98//! ## Filtering for a Specific Version
99//!
100//! ```rust
101//! use netflow_parser::{NetflowParser, NetflowPacket};
102//!
103//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
104//! let parsed = NetflowParser::default().parse_bytes(&v5_packet);
105//!
106//! let v5_parsed: Vec<NetflowPacket> = parsed.into_iter().filter(|p| p.is_v5()).collect();
107//! ```
108//!
109//! ## Iterator API
110//!
111//! For high-performance scenarios where you want to avoid allocating a `Vec`, you can use the iterator API to process packets one-by-one as they're parsed:
112//!
113//! ```rust
114//! use netflow_parser::{NetflowParser, NetflowPacket};
115//!
116//! # let buffer = [0u8; 72];
117//! let mut parser = NetflowParser::default();
118//!
119//! // Process packets without collecting into a Vec
120//! for packet in parser.iter_packets(&buffer) {
121//! match packet {
122//! NetflowPacket::V5(v5) => {
123//! // Process V5 packet
124//! println!("V5 packet from {}", v5.header.version);
125//! }
126//! NetflowPacket::V9(v9) => {
127//! // Process V9 packet
128//! for flowset in &v9.flowsets {
129//! // Handle flowsets
130//! }
131//! }
132//! NetflowPacket::IPFix(ipfix) => {
133//! // Process IPFIX packet
134//! }
135//! NetflowPacket::Error(e) => {
136//! eprintln!("Parse error: {:?}", e);
137//! }
138//! _ => {}
139//! }
140//! }
141//! ```
142//!
143//! The iterator provides access to unconsumed bytes for advanced use cases:
144//!
145//! ```rust
146//! use netflow_parser::NetflowParser;
147//!
148//! # let buffer = [0u8; 72];
149//! let mut parser = NetflowParser::default();
150//! let mut iter = parser.iter_packets(&buffer);
151//!
152//! while let Some(packet) = iter.next() {
153//! // Process packet
154//! # _ = packet;
155//! }
156//!
157//! // Check if all bytes were consumed
158//! if !iter.is_complete() {
159//! println!("Warning: {} bytes remain unconsumed", iter.remaining().len());
160//! }
161//! ```
162//!
163//! ### Benefits of Iterator API
164//!
165//! - **Zero allocation**: Packets are yielded one-by-one without allocating a `Vec`
166//! - **Memory efficient**: Ideal for processing large batches or continuous streams
167//! - **Lazy evaluation**: Only parses packets as you consume them
168//! - **Template caching preserved**: V9/IPFIX template state is maintained across iterations
169//! - **Composable**: Works with standard Rust iterator methods (`.filter()`, `.map()`, `.take()`, etc.)
170//! - **Buffer inspection**: Access unconsumed bytes via `.remaining()` and check completion with `.is_complete()`
171//!
172//! ### Iterator Examples
173//!
174//! ```rust
175//! # use netflow_parser::{NetflowParser, NetflowPacket};
176//! # let buffer = [0u8; 72];
177//! # let mut parser = NetflowParser::default();
178//! // Count V5 packets without collecting
179//! let count = parser.iter_packets(&buffer)
180//! .filter(|p| p.is_v5())
181//! .count();
182//!
183//! // Process only the first 10 packets
184//! for packet in parser.iter_packets(&buffer).take(10) {
185//! // Handle packet
186//! # _ = packet;
187//! }
188//!
189//! // Collect only if needed (equivalent to parse_bytes())
190//! let packets: Vec<_> = parser.iter_packets(&buffer).collect();
191//!
192//! // Check unconsumed bytes (useful for mixed protocol streams)
193//! let mut iter = parser.iter_packets(&buffer);
194//! for packet in &mut iter {
195//! // Process packet
196//! # _ = packet;
197//! }
198//! if !iter.is_complete() {
199//! let remaining = iter.remaining();
200//! // Handle non-netflow data at end of buffer
201//! # _ = remaining;
202//! }
203//! ```
204//!
205//! ## Parsing Out Unneeded Versions
206//! If you only care about a specific version or versions you can specify `allowed_versions`:
207//! ```rust
208//! use netflow_parser::{NetflowParser, NetflowPacket};
209//!
210//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
211//! let mut parser = NetflowParser::default();
212//! parser.allowed_versions = [7, 9].into();
213//! let parsed = parser.parse_bytes(&v5_packet);
214//! ```
215//!
216//! This code will return an empty Vec as version 5 is not allowed.
217//!
218//! ## Error Handling Configuration
219//!
220//! To prevent memory exhaustion from malformed packets, the parser limits the size of error buffer samples. By default, only the first 256 bytes of unparseable data are stored in error messages. You can customize this limit for all parsers:
221//!
222//! ```rust
223//! use netflow_parser::NetflowParser;
224//!
225//! let mut parser = NetflowParser::default();
226//!
227//! // Configure maximum error buffer size for the main parser (default: 256 bytes)
228//! // This applies to generic parsing errors
229//! parser.max_error_sample_size = 512;
230//!
231//! // Configure maximum error buffer size for V9 (default: 256 bytes)
232//! parser.v9_parser.max_error_sample_size = 512;
233//!
234//! // Configure maximum error buffer size for IPFIX (default: 256 bytes)
235//! parser.ipfix_parser.max_error_sample_size = 512;
236//!
237//! # let some_packet = [0u8; 72];
238//! let parsed = parser.parse_bytes(&some_packet);
239//! ```
240//!
241//! This setting helps prevent memory exhaustion when processing malformed or malicious packets while still providing enough context for debugging.
242//!
243//! ## Netflow Common
244//!
245//! We have included a `NetflowCommon` and `NetflowCommonFlowSet` structure.
246//! This will allow you to use common fields without unpacking values from specific versions.
247//! If the packet flow does not have the matching field it will simply be left as `None`.
248//!
249//! ### NetflowCommon and NetflowCommonFlowSet Struct:
250//! ```rust
251//! use std::net::IpAddr;
252//! use netflow_parser::protocol::ProtocolTypes;
253//!
254//! #[derive(Debug, Default)]
255//! pub struct NetflowCommon {
256//! pub version: u16,
257//! pub timestamp: u32,
258//! pub flowsets: Vec<NetflowCommonFlowSet>,
259//! }
260//!
261//! #[derive(Debug, Default)]
262//! struct NetflowCommonFlowSet {
263//! src_addr: Option<IpAddr>,
264//! dst_addr: Option<IpAddr>,
265//! src_port: Option<u16>,
266//! dst_port: Option<u16>,
267//! protocol_number: Option<u8>,
268//! protocol_type: Option<ProtocolTypes>,
269//! first_seen: Option<u32>,
270//! last_seen: Option<u32>,
271//! src_mac: Option<String>,
272//! dst_mac: Option<String>,
273//! }
274//! ```
275//!
276//! ### Converting NetflowPacket to NetflowCommon
277//!
278//! ```rust,ignore
279//! use netflow_parser::{NetflowParser, NetflowPacket};
280//!
281//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3,
282//! 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1,
283//! 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7];
284//! let netflow_common = NetflowParser::default()
285//! .parse_bytes(&v5_packet)
286//! .first()
287//! .unwrap()
288//! .as_netflow_common()
289//! .unwrap();
290//!
291//! for common_flow in netflow_common.flowsets.iter() {
292//! println!("Src Addr: {} Dst Addr: {}", common_flow.src_addr.unwrap(), common_flow.dst_addr.unwrap());
293//! }
294//! ```
295//!
296//! ### Flattened flowsets
297//!
298//! To gather all flowsets from all packets into a flattened vector:
299//!
300//! ```rust,ignore
301//! use netflow_parser::NetflowParser;
302//!
303//! # let v5_packet = [0u8; 72];
304//! let flowsets = NetflowParser::default().parse_bytes_as_netflow_common_flowsets(&v5_packet);
305//! ```
306//!
307//! ### Custom Field Mappings for V9 and IPFIX
308//!
309//! By default, NetflowCommon maps standard IANA fields to the common structure. However, you can customize which fields are used for V9 and IPFIX packets using configuration structs. This is useful when:
310//!
311//! - You want to prefer IPv6 addresses over IPv4
312//! - Your vendor uses non-standard field mappings
313//! - You need to extract data from vendor-specific enterprise fields
314//!
315//! #### V9 Custom Field Mapping
316//!
317//! ```rust,ignore
318//! use netflow_parser::netflow_common::{NetflowCommon, V9FieldMappingConfig};
319//! use netflow_parser::variable_versions::v9_lookup::V9Field;
320//!
321//! // Create a custom configuration that prefers IPv6 addresses
322//! let mut config = V9FieldMappingConfig::default();
323//! config.src_addr.primary = V9Field::Ipv6SrcAddr;
324//! config.src_addr.fallback = Some(V9Field::Ipv4SrcAddr);
325//! config.dst_addr.primary = V9Field::Ipv6DstAddr;
326//! config.dst_addr.fallback = Some(V9Field::Ipv4DstAddr);
327//!
328//! // Use with a parsed V9 packet
329//! // let common = NetflowCommon::from_v9_with_config(&v9_packet, &config);
330//! ```
331//!
332//! #### IPFIX Custom Field Mapping
333//!
334//! ```rust,ignore
335//! use netflow_parser::netflow_common::{NetflowCommon, IPFixFieldMappingConfig};
336//! use netflow_parser::variable_versions::ipfix_lookup::{IPFixField, IANAIPFixField};
337//!
338//! // Create a custom configuration that prefers IPv6 addresses
339//! let mut config = IPFixFieldMappingConfig::default();
340//! config.src_addr.primary = IPFixField::IANA(IANAIPFixField::SourceIpv6address);
341//! config.src_addr.fallback = Some(IPFixField::IANA(IANAIPFixField::SourceIpv4address));
342//! config.dst_addr.primary = IPFixField::IANA(IANAIPFixField::DestinationIpv6address);
343//! config.dst_addr.fallback = Some(IPFixField::IANA(IANAIPFixField::DestinationIpv4address));
344//!
345//! // Use with a parsed IPFIX packet
346//! // let common = NetflowCommon::from_ipfix_with_config(&ipfix_packet, &config);
347//! ```
348//!
349//! #### Available Configuration Fields
350//!
351//! Both `V9FieldMappingConfig` and `IPFixFieldMappingConfig` support configuring:
352//!
353//! | Field | Description | Default V9 Field | Default IPFIX Field |
354//! |-------|-------------|------------------|---------------------|
355//! | `src_addr` | Source IP address | Ipv4SrcAddr (fallback: Ipv6SrcAddr) | SourceIpv4address (fallback: SourceIpv6address) |
356//! | `dst_addr` | Destination IP address | Ipv4DstAddr (fallback: Ipv6DstAddr) | DestinationIpv4address (fallback: DestinationIpv6address) |
357//! | `src_port` | Source port | L4SrcPort | SourceTransportPort |
358//! | `dst_port` | Destination port | L4DstPort | DestinationTransportPort |
359//! | `protocol` | Protocol number | Protocol | ProtocolIdentifier |
360//! | `first_seen` | Flow start time | FirstSwitched | FlowStartSysUpTime |
361//! | `last_seen` | Flow end time | LastSwitched | FlowEndSysUpTime |
362//! | `src_mac` | Source MAC address | InSrcMac | SourceMacaddress |
363//! | `dst_mac` | Destination MAC address | InDstMac | DestinationMacaddress |
364//!
365//! Each field mapping has a `primary` field (always checked first) and an optional `fallback` field (used if primary is not present in the flow record).
366//!
367//! ## Re-Exporting Flows
368//!
369//! Parsed V5, V7, V9, and IPFIX packets can be re-exported back into bytes.
370//!
371//! **V9/IPFIX Padding Behavior:**
372//! - For **parsed packets**: Original padding is preserved exactly for byte-perfect round-trips
373//! - For **manually created packets**: Padding is automatically calculated to align FlowSets to 4-byte boundaries
374//!
375//! **Creating Data Structs:**
376//! For convenience, use `Data::new()` and `OptionsData::new()` to create data structures without manually specifying padding:
377//!
378//! ```rust,ignore
379//! use netflow_parser::variable_versions::ipfix::Data;
380//!
381//! // Padding is automatically set to empty vec and calculated during export
382//! let data = Data::new(vec![vec![
383//! (field1, value1),
384//! (field2, value2),
385//! ]]);
386//! ```
387//!
388//! See `examples/manual_ipfix_creation.rs` for a complete example of creating IPFIX packets from scratch.
389//!
390//! ```rust
391//! use netflow_parser::{NetflowParser, NetflowPacket};
392//!
393//! let packet = [
394//! 0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3,
395//! 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1,
396//! 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,
397//! ];
398//! if let NetflowPacket::V5(v5) = NetflowParser::default()
399//! .parse_bytes(&packet)
400//! .first()
401//! .unwrap()
402//! {
403//! assert_eq!(v5.to_be_bytes(), packet);
404//! }
405//! ```
406//!
407//! ## Template Cache Configuration
408//!
409//! V9 and IPFIX parsers use LRU (Least Recently Used) caching to store templates with a configurable size limit. This prevents memory exhaustion from template flooding attacks while maintaining good performance for legitimate traffic.
410//!
411//! ### Default Behavior
412//!
413//! By default, parsers cache up to 1000 templates:
414//!
415//! ```rust
416//! use netflow_parser::NetflowParser;
417//!
418//! // Uses default cache size of 1000 templates per parser
419//! let parser = NetflowParser::default();
420//! ```
421//!
422//! ### Custom Cache Size
423//!
424//! Use the builder pattern to configure cache sizes:
425//!
426//! ```rust
427//! use netflow_parser::NetflowParser;
428//!
429//! // Configure both V9 and IPFIX with the same cache size
430//! let parser = NetflowParser::builder()
431//! .with_cache_size(2000)
432//! .build()
433//! .expect("Failed to build parser");
434//!
435//! // Configure V9 and IPFIX independently
436//! let parser = NetflowParser::builder()
437//! .with_v9_cache_size(1000)
438//! .with_ipfix_cache_size(5000)
439//! .build()
440//! .expect("Failed to build parser");
441//! ```
442//!
443//! ### Cache Behavior
444//!
445//! - When the cache is full, the least recently used template is evicted
446//! - Templates are keyed by template ID (per source)
447//! - Each parser instance maintains its own template cache
448//! - For multi-source deployments, create separate parser instances per source
449//!
450//! ### Template TTL (Time-to-Live)
451//!
452//! Optionally configure templates to expire after a time duration. This is useful for:
453//! - Handling exporters that reuse template IDs with different schemas
454//! - Forcing periodic template refresh from exporters
455//! - Testing template re-learning behavior
456//!
457//! **Note:** TTL is disabled by default. Templates persist until LRU eviction unless explicitly configured.
458//!
459//! #### Configuration Examples
460//!
461//! ```rust
462//! use netflow_parser::NetflowParser;
463//! use netflow_parser::variable_versions::ttl::TtlConfig;
464//! use std::time::Duration;
465//!
466//! // Templates expire after 2 hours
467//! let parser = NetflowParser::builder()
468//! .with_cache_size(1000)
469//! .with_ttl(TtlConfig::new(Duration::from_secs(2 * 3600)))
470//! .build()
471//! .unwrap();
472//!
473//! // Using default TTL (2 hours)
474//! let parser = NetflowParser::builder()
475//! .with_cache_size(1000)
476//! .with_ttl(TtlConfig::default())
477//! .build()
478//! .unwrap();
479//!
480//! // Different TTL for V9 and IPFIX
481//! let parser = NetflowParser::builder()
482//! .with_v9_ttl(TtlConfig::new(Duration::from_secs(3600)))
483//! .with_ipfix_ttl(TtlConfig::new(Duration::from_secs(2 * 3600)))
484//! .build()
485//! .unwrap();
486//! ```
487//!
488//! ## V9/IPFIX Notes
489//!
490//! Parse the data (`&[u8]`) like any other version. The parser (`NetflowParser`) caches parsed templates using LRU eviction, so you can send header/data flowset combos and it will use the cached templates. Templates are automatically cached and evicted when the cache limit is reached.
491//!
492//! **Template Cache Introspection:**
493//! Use the introspection methods to inspect template cache state without affecting LRU ordering:
494//!
495//! ```rust
496//! use netflow_parser::NetflowParser;
497//! let parser = NetflowParser::default();
498//!
499//! // Get cache statistics
500//! let stats = parser.v9_cache_stats();
501//! println!("V9 cache: {}/{} templates", stats.current_size, stats.max_size);
502//!
503//! // List all cached template IDs
504//! let template_ids = parser.v9_template_ids();
505//! println!("Cached templates: {:?}", template_ids);
506//!
507//! // Check if a specific template exists (doesn't affect LRU)
508//! if parser.has_v9_template(256) {
509//! println!("Template 256 is cached");
510//! }
511//! ```
512//!
513//! **IPFIX Note:** We only parse sequence number and domain id, it is up to you if you wish to validate it.
514//!
515//! To access templates flowset of a processed V9/IPFix flowset you can find the `flowsets` attribute on the Parsed Record. In there you can find `Templates`, `Option Templates`, and `Data` Flowsets.
516//!
517//! ## Performance & Thread Safety
518//!
519//! ### Thread Safety
520//!
521//! Parsers (`NetflowParser`, `V9Parser`, `IPFixParser`) are **not thread-safe** and should not be shared across threads without external synchronization. Each parser maintains internal state (template caches) that is modified during parsing.
522//!
523//! **Recommended pattern for multi-threaded applications:**
524//! - Create one parser instance per thread
525//! - Each thread processes packets from a single router/source
526//! - See `examples/netflow_udp_listener_multi_threaded.rs` for implementation example
527//!
528//! ### Performance Optimizations
529//!
530//! This library includes several performance optimizations:
531//!
532//! 1. **Single-pass field caching** - NetflowCommon conversions use efficient single-pass lookups
533//! 2. **Minimal cloning** - Template storage avoids unnecessary vector clones
534//! 3. **Optimized string processing** - Single-pass filtering and prefix stripping
535//! 4. **Capacity pre-allocation** - Vectors pre-allocate when sizes are known
536//! 5. **Bounded error buffers** - Error handling limits memory consumption to prevent exhaustion
537//!
538//! **Best practices for optimal performance:**
539//! - Reuse parser instances instead of creating new ones for each packet
540//! - Use `iter_packets()` instead of `parse_bytes()` when you don't need all packets in a Vec
541//! - Use `parse_bytes_as_netflow_common_flowsets()` when you only need flow data
542//! - For V9/IPFIX, batch process packets from the same source to maximize template cache hits
543//!
544//! ## Features
545//!
546//! * `parse_unknown_fields` - When enabled fields not listed in this library will attempt to be parsed as a Vec of bytes and the field_number listed. When disabled an error is thrown when attempting to parse those fields. Enabled by default.
547//! * `netflow_common` - When enabled provides `NetflowCommon` and `NetflowCommonFlowSet` structures for working with common fields across different Netflow versions. Disabled by default.
548//!
549//! ## Included Examples
550//!
551//! Examples have been included mainly for those who want to use this parser to read from a Socket and parse netflow. In those cases with V9/IPFix it is best to create a new parser for each router. There are both single threaded and multi-threaded examples in the examples directory.
552//!
553//! Examples that listen on a specific port use 9995 by default, however netflow can be configurated to use a variety of URP ports:
554//! * **2055**: The most widely recognized default for NetFlow.
555//! * **9995 / 9996**: Popular alternatives, especially with Cisco devices.
556//! * **9025, 9026**: Other recognized port options.
557//! * **6343**: The default for sFlow, often used alongside NetFlow.
558//! * **4739**: The default port for IPFIX (a NetFlow successor).
559//!
560//! To run:
561//!
562//! ```cargo run --example netflow_udp_listener_multi_threaded```
563//!
564//! ```cargo run --example netflow_udp_listener_single_threaded```
565//!
566//! ```cargo run --example netflow_udp_listener_tokio```
567//!
568//! ```cargo run --example netflow_pcap```
569//!
570//! ```cargo run --example manual_ipfix_creation```
571//!
572//! The pcap example also shows how to cache flows that have not yet discovered a template.
573//!
574//! ## Support My Work
575//!
576//! If you find my work helpful, consider supporting me!
577//!
578//! [](https://ko-fi.com/michaelmileusnich)
579//!
580//! [](https://github.com/sponsors/mikemiles-dev)
581
582#[cfg(feature = "netflow_common")]
583pub mod netflow_common;
584pub mod protocol;
585pub mod static_versions;
586mod tests;
587pub mod variable_versions;
588
589#[cfg(feature = "netflow_common")]
590use crate::netflow_common::{NetflowCommon, NetflowCommonError, NetflowCommonFlowSet};
591
592use static_versions::{
593 v5::{V5, V5Parser},
594 v7::{V7, V7Parser},
595};
596use variable_versions::Config;
597use variable_versions::enterprise_registry::EnterpriseFieldDef;
598use variable_versions::ipfix::{IPFix, IPFixParser};
599use variable_versions::v9::{V9, V9Parser};
600
601use nom_derive::{Nom, Parse};
602use serde::Serialize;
603
604use std::collections::HashSet;
605
606/// Enum of supported Netflow Versions
607#[derive(Debug, Clone, Serialize)]
608pub enum NetflowPacket {
609 /// Version 5
610 V5(V5),
611 /// Version 7
612 V7(V7),
613 /// Version 9
614 V9(V9),
615 /// IPFix
616 IPFix(IPFix),
617 /// Error
618 Error(NetflowPacketError),
619}
620
621impl NetflowPacket {
622 pub fn is_v5(&self) -> bool {
623 matches!(self, Self::V5(_v))
624 }
625 pub fn is_v7(&self) -> bool {
626 matches!(self, Self::V7(_v))
627 }
628 pub fn is_v9(&self) -> bool {
629 matches!(self, Self::V9(_v))
630 }
631 pub fn is_ipfix(&self) -> bool {
632 matches!(self, Self::IPFix(_v))
633 }
634 pub fn is_error(&self) -> bool {
635 matches!(self, Self::Error(_v))
636 }
637 #[cfg(feature = "netflow_common")]
638 pub fn as_netflow_common(&self) -> Result<NetflowCommon, NetflowCommonError> {
639 self.try_into()
640 }
641}
642
643#[derive(Nom)]
644/// Generic Netflow Header for shared versions
645struct GenericNetflowHeader {
646 version: u16,
647}
648
649/// Main parser for Netflow packets supporting V5, V7, V9, and IPFIX.
650///
651/// Use [`NetflowParser::builder()`] for ergonomic configuration with the builder pattern,
652/// or [`NetflowParser::default()`] for quick setup with defaults.
653///
654/// # Examples
655///
656/// ```rust
657/// use netflow_parser::NetflowParser;
658/// use netflow_parser::variable_versions::ttl::TtlConfig;
659/// use std::time::Duration;
660///
661/// // Using builder pattern (recommended)
662/// let parser = NetflowParser::builder()
663/// .with_cache_size(2000)
664/// .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
665/// .build()
666/// .expect("Failed to build parser");
667///
668/// // Using default
669/// let parser = NetflowParser::default();
670/// ```
671#[derive(Debug)]
672pub struct NetflowParser {
673 pub v9_parser: V9Parser,
674 pub ipfix_parser: IPFixParser,
675 pub allowed_versions: HashSet<u16>,
676 /// Maximum number of bytes to include in error samples to prevent memory exhaustion.
677 /// Defaults to 256 bytes.
678 pub max_error_sample_size: usize,
679}
680
681/// Statistics about template cache utilization.
682#[derive(Debug, Clone)]
683pub struct CacheStats {
684 /// Current number of cached templates
685 pub current_size: usize,
686 /// Maximum cache size before LRU eviction
687 pub max_size: usize,
688 /// TTL configuration (if enabled)
689 pub ttl_config: Option<variable_versions::ttl::TtlConfig>,
690}
691
692/// Builder for configuring and constructing a [`NetflowParser`].
693///
694/// # Examples
695///
696/// ```rust
697/// use netflow_parser::NetflowParser;
698/// use netflow_parser::variable_versions::ttl::TtlConfig;
699/// use std::collections::HashSet;
700/// use std::time::Duration;
701///
702/// let parser = NetflowParser::builder()
703/// .with_cache_size(2000)
704/// .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
705/// .with_allowed_versions([5, 9, 10].into())
706/// .with_max_error_sample_size(512)
707/// .build()
708/// .expect("Failed to build parser");
709/// ```
710#[derive(Debug, Clone)]
711pub struct NetflowParserBuilder {
712 v9_config: Config,
713 ipfix_config: Config,
714 allowed_versions: HashSet<u16>,
715 max_error_sample_size: usize,
716}
717
718impl Default for NetflowParserBuilder {
719 fn default() -> Self {
720 Self {
721 v9_config: Config::new(1000, None),
722 ipfix_config: Config::new(1000, None),
723 allowed_versions: [5, 7, 9, 10].iter().cloned().collect(),
724 max_error_sample_size: 256,
725 }
726 }
727}
728
729impl NetflowParserBuilder {
730 /// Sets the template cache size for both V9 and IPFIX parsers.
731 ///
732 /// # Arguments
733 ///
734 /// * `size` - Maximum number of templates to cache (must be > 0)
735 ///
736 /// # Examples
737 ///
738 /// ```rust
739 /// use netflow_parser::NetflowParser;
740 ///
741 /// let parser = NetflowParser::builder()
742 /// .with_cache_size(2000)
743 /// .build()
744 /// .expect("Failed to build parser");
745 /// ```
746 pub fn with_cache_size(mut self, size: usize) -> Self {
747 self.v9_config.max_template_cache_size = size;
748 self.ipfix_config.max_template_cache_size = size;
749 self
750 }
751
752 /// Sets the V9 parser template cache size independently.
753 ///
754 /// # Arguments
755 ///
756 /// * `size` - Maximum number of templates to cache (must be > 0)
757 pub fn with_v9_cache_size(mut self, size: usize) -> Self {
758 self.v9_config.max_template_cache_size = size;
759 self
760 }
761
762 /// Sets the IPFIX parser template cache size independently.
763 ///
764 /// * `size` - Maximum number of templates to cache (must be > 0)
765 pub fn with_ipfix_cache_size(mut self, size: usize) -> Self {
766 self.ipfix_config.max_template_cache_size = size;
767 self
768 }
769
770 /// Sets the TTL configuration for both V9 and IPFIX parsers.
771 ///
772 /// # Arguments
773 ///
774 /// * `ttl` - TTL configuration (time-based)
775 ///
776 /// # Examples
777 ///
778 /// ```rust
779 /// use netflow_parser::NetflowParser;
780 /// use netflow_parser::variable_versions::ttl::TtlConfig;
781 /// use std::time::Duration;
782 ///
783 /// let parser = NetflowParser::builder()
784 /// .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
785 /// .build()
786 /// .expect("Failed to build parser");
787 /// ```
788 pub fn with_ttl(mut self, ttl: variable_versions::ttl::TtlConfig) -> Self {
789 self.v9_config.ttl_config = Some(ttl.clone());
790 self.ipfix_config.ttl_config = Some(ttl);
791 self
792 }
793
794 /// Sets the TTL configuration for V9 parser independently.
795 pub fn with_v9_ttl(mut self, ttl: variable_versions::ttl::TtlConfig) -> Self {
796 self.v9_config.ttl_config = Some(ttl);
797 self
798 }
799
800 /// Sets the TTL configuration for IPFIX parser independently.
801 pub fn with_ipfix_ttl(mut self, ttl: variable_versions::ttl::TtlConfig) -> Self {
802 self.ipfix_config.ttl_config = Some(ttl);
803 self
804 }
805
806 /// Sets which Netflow versions are allowed to be parsed.
807 ///
808 /// # Arguments
809 ///
810 /// * `versions` - Set of allowed version numbers (5, 7, 9, 10)
811 ///
812 /// # Examples
813 ///
814 /// ```rust
815 /// use netflow_parser::NetflowParser;
816 ///
817 /// // Only parse V9 and IPFIX
818 /// let parser = NetflowParser::builder()
819 /// .with_allowed_versions([9, 10].into())
820 /// .build()
821 /// .expect("Failed to build parser");
822 /// ```
823 pub fn with_allowed_versions(mut self, versions: HashSet<u16>) -> Self {
824 self.allowed_versions = versions;
825 self
826 }
827
828 /// Sets the maximum error sample size for error reporting.
829 ///
830 /// # Arguments
831 ///
832 /// * `size` - Maximum bytes to include in error messages
833 ///
834 /// # Examples
835 ///
836 /// ```rust
837 /// use netflow_parser::NetflowParser;
838 ///
839 /// let parser = NetflowParser::builder()
840 /// .with_max_error_sample_size(512)
841 /// .build()
842 /// .expect("Failed to build parser");
843 /// ```
844 pub fn with_max_error_sample_size(mut self, size: usize) -> Self {
845 self.max_error_sample_size = size;
846 self
847 }
848
849 /// Registers a custom enterprise field definition for both V9 and IPFIX parsers.
850 ///
851 /// This allows library users to define their own enterprise-specific fields without
852 /// modifying the library source code. Registered fields will be parsed according to
853 /// their specified data type instead of falling back to raw bytes.
854 ///
855 /// # Arguments
856 ///
857 /// * `def` - Enterprise field definition containing enterprise number, field number, name, and data type
858 ///
859 /// # Examples
860 ///
861 /// ```rust
862 /// use netflow_parser::NetflowParser;
863 /// use netflow_parser::variable_versions::enterprise_registry::EnterpriseFieldDef;
864 /// use netflow_parser::variable_versions::data_number::FieldDataType;
865 ///
866 /// let parser = NetflowParser::builder()
867 /// .register_enterprise_field(EnterpriseFieldDef::new(
868 /// 12345, // Enterprise number
869 /// 1, // Field number
870 /// "customMetric",
871 /// FieldDataType::UnsignedDataNumber,
872 /// ))
873 /// .register_enterprise_field(EnterpriseFieldDef::new(
874 /// 12345,
875 /// 2,
876 /// "customApplicationName",
877 /// FieldDataType::String,
878 /// ))
879 /// .build()
880 /// .expect("Failed to build parser");
881 /// ```
882 pub fn register_enterprise_field(mut self, def: EnterpriseFieldDef) -> Self {
883 self.v9_config.enterprise_registry.register(def.clone());
884 self.ipfix_config.enterprise_registry.register(def);
885 self
886 }
887
888 /// Registers multiple custom enterprise field definitions at once.
889 ///
890 /// # Arguments
891 ///
892 /// * `defs` - Iterator of enterprise field definitions
893 ///
894 /// # Examples
895 ///
896 /// ```rust
897 /// use netflow_parser::NetflowParser;
898 /// use netflow_parser::variable_versions::enterprise_registry::EnterpriseFieldDef;
899 /// use netflow_parser::variable_versions::data_number::FieldDataType;
900 ///
901 /// let fields = vec![
902 /// EnterpriseFieldDef::new(12345, 1, "field1", FieldDataType::UnsignedDataNumber),
903 /// EnterpriseFieldDef::new(12345, 2, "field2", FieldDataType::String),
904 /// EnterpriseFieldDef::new(12345, 3, "field3", FieldDataType::Ip4Addr),
905 /// ];
906 ///
907 /// let parser = NetflowParser::builder()
908 /// .register_enterprise_fields(fields)
909 /// .build()
910 /// .expect("Failed to build parser");
911 /// ```
912 pub fn register_enterprise_fields(
913 mut self,
914 defs: impl IntoIterator<Item = EnterpriseFieldDef>,
915 ) -> Self {
916 for def in defs {
917 self.v9_config.enterprise_registry.register(def.clone());
918 self.ipfix_config.enterprise_registry.register(def);
919 }
920 self
921 }
922
923 /// Builds the configured [`NetflowParser`].
924 ///
925 /// # Errors
926 ///
927 /// Returns an error if:
928 /// - Template cache size is 0
929 /// - Parser initialization fails
930 ///
931 /// # Examples
932 ///
933 /// ```rust
934 /// use netflow_parser::NetflowParser;
935 ///
936 /// let parser = NetflowParser::builder()
937 /// .with_cache_size(2000)
938 /// .build()
939 /// .expect("Failed to build parser");
940 /// ```
941 pub fn build(self) -> Result<NetflowParser, String> {
942 let v9_parser =
943 V9Parser::try_new(self.v9_config).map_err(|e| format!("V9 parser error: {}", e))?;
944 let ipfix_parser = IPFixParser::try_new(self.ipfix_config)
945 .map_err(|e| format!("IPFIX parser error: {}", e))?;
946
947 Ok(NetflowParser {
948 v9_parser,
949 ipfix_parser,
950 allowed_versions: self.allowed_versions,
951 max_error_sample_size: self.max_error_sample_size,
952 })
953 }
954}
955
956#[derive(Debug, Clone)]
957pub enum ParsedNetflow<'a> {
958 Success {
959 packet: NetflowPacket,
960 remaining: &'a [u8],
961 },
962 Error {
963 error: NetflowParseError,
964 },
965 UnallowedVersion,
966}
967
968#[derive(Debug, Clone, Serialize)]
969pub struct NetflowPacketError {
970 pub error: NetflowParseError,
971 pub remaining: Vec<u8>,
972}
973
974#[derive(Debug, Clone, Serialize)]
975pub enum NetflowParseError {
976 Incomplete(String),
977 Partial(PartialParse),
978 UnknownVersion(Vec<u8>),
979}
980
981#[derive(Debug, Clone, Serialize)]
982pub struct PartialParse {
983 pub version: u16,
984 pub remaining: Vec<u8>,
985 pub error: String,
986}
987
988/// Iterator that yields NetflowPacket items from a byte buffer without allocating a Vec.
989/// Maintains parser state for template caching (V9/IPFIX).
990pub struct NetflowPacketIterator<'a> {
991 parser: &'a mut NetflowParser,
992 remaining: &'a [u8],
993 errored: bool,
994}
995
996impl<'a> NetflowPacketIterator<'a> {
997 /// Returns the unconsumed bytes remaining in the buffer.
998 ///
999 /// This is useful for:
1000 /// - Debugging: See how much data was consumed
1001 /// - Mixed protocols: Process non-netflow data after netflow packets
1002 /// - Resumption: Know where parsing stopped
1003 ///
1004 /// # Examples
1005 ///
1006 /// ```rust
1007 /// use netflow_parser::NetflowParser;
1008 ///
1009 /// let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
1010 /// let mut parser = NetflowParser::default();
1011 /// let mut iter = parser.iter_packets(&v5_packet);
1012 ///
1013 /// while let Some(_packet) = iter.next() {
1014 /// // Process packet
1015 /// }
1016 ///
1017 /// // Check how many bytes remain unconsumed
1018 /// assert_eq!(iter.remaining().len(), 0);
1019 /// ```
1020 pub fn remaining(&self) -> &'a [u8] {
1021 self.remaining
1022 }
1023
1024 /// Returns true if all bytes have been consumed or an error occurred.
1025 ///
1026 /// This is useful for validation and ensuring complete buffer processing.
1027 ///
1028 /// # Examples
1029 ///
1030 /// ```rust
1031 /// use netflow_parser::NetflowParser;
1032 ///
1033 /// let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
1034 /// let mut parser = NetflowParser::default();
1035 /// let mut iter = parser.iter_packets(&v5_packet);
1036 ///
1037 /// // Consume all packets
1038 /// for _packet in &mut iter {
1039 /// // Process packet
1040 /// }
1041 ///
1042 /// assert!(iter.is_complete());
1043 /// ```
1044 pub fn is_complete(&self) -> bool {
1045 self.remaining.is_empty() || self.errored
1046 }
1047}
1048
1049impl<'a> Iterator for NetflowPacketIterator<'a> {
1050 type Item = NetflowPacket;
1051
1052 fn next(&mut self) -> Option<Self::Item> {
1053 // Stop if we've errored or no bytes remain
1054 if self.errored || self.remaining.is_empty() {
1055 return None;
1056 }
1057
1058 match self.parser.parse_packet_by_version(self.remaining) {
1059 ParsedNetflow::Success {
1060 packet,
1061 remaining: new_remaining,
1062 } => {
1063 self.remaining = new_remaining;
1064 Some(packet)
1065 }
1066 ParsedNetflow::UnallowedVersion => {
1067 self.errored = true;
1068 None
1069 }
1070 ParsedNetflow::Error { error } => {
1071 self.errored = true;
1072 // Only include first N bytes of remaining data in error to prevent memory exhaustion
1073 let remaining_sample =
1074 if self.remaining.len() > self.parser.max_error_sample_size {
1075 self.remaining[..self.parser.max_error_sample_size].to_vec()
1076 } else {
1077 self.remaining.to_vec()
1078 };
1079 Some(NetflowPacket::Error(NetflowPacketError {
1080 error,
1081 remaining: remaining_sample,
1082 }))
1083 }
1084 }
1085 }
1086}
1087
1088impl Default for NetflowParser {
1089 fn default() -> Self {
1090 Self {
1091 v9_parser: V9Parser::default(),
1092 ipfix_parser: IPFixParser::default(),
1093 allowed_versions: [5, 7, 9, 10].iter().cloned().collect(),
1094 max_error_sample_size: 256,
1095 }
1096 }
1097}
1098
1099impl NetflowParser {
1100 /// Creates a new builder for configuring a [`NetflowParser`].
1101 ///
1102 /// # Examples
1103 ///
1104 /// ```rust
1105 /// use netflow_parser::NetflowParser;
1106 /// use netflow_parser::variable_versions::ttl::TtlConfig;
1107 /// use std::time::Duration;
1108 ///
1109 /// let parser = NetflowParser::builder()
1110 /// .with_cache_size(2000)
1111 /// .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
1112 /// .build()
1113 /// .expect("Failed to build parser");
1114 /// ```
1115 pub fn builder() -> NetflowParserBuilder {
1116 NetflowParserBuilder::default()
1117 }
1118
1119 /// Gets statistics about the V9 template cache.
1120 ///
1121 /// # Examples
1122 ///
1123 /// ```rust
1124 /// use netflow_parser::NetflowParser;
1125 ///
1126 /// let parser = NetflowParser::default();
1127 /// let stats = parser.v9_cache_stats();
1128 /// println!("V9 cache: {}/{} templates", stats.current_size, stats.max_size);
1129 /// ```
1130 pub fn v9_cache_stats(&self) -> CacheStats {
1131 CacheStats {
1132 current_size: self.v9_parser.templates.len()
1133 + self.v9_parser.options_templates.len(),
1134 max_size: self.v9_parser.max_template_cache_size,
1135 ttl_config: self.v9_parser.ttl_config.clone(),
1136 }
1137 }
1138
1139 /// Gets statistics about the IPFIX template cache.
1140 ///
1141 /// # Examples
1142 ///
1143 /// ```rust
1144 /// use netflow_parser::NetflowParser;
1145 ///
1146 /// let parser = NetflowParser::default();
1147 /// let stats = parser.ipfix_cache_stats();
1148 /// println!("IPFIX cache: {}/{} templates", stats.current_size, stats.max_size);
1149 /// ```
1150 pub fn ipfix_cache_stats(&self) -> CacheStats {
1151 CacheStats {
1152 current_size: self.ipfix_parser.templates.len()
1153 + self.ipfix_parser.v9_templates.len()
1154 + self.ipfix_parser.ipfix_options_templates.len()
1155 + self.ipfix_parser.v9_options_templates.len(),
1156 max_size: self.ipfix_parser.max_template_cache_size,
1157 ttl_config: self.ipfix_parser.ttl_config.clone(),
1158 }
1159 }
1160
1161 /// Lists all cached V9 template IDs.
1162 ///
1163 /// Note: This returns template IDs from both regular and options templates.
1164 ///
1165 /// # Examples
1166 ///
1167 /// ```rust
1168 /// use netflow_parser::NetflowParser;
1169 ///
1170 /// let parser = NetflowParser::default();
1171 /// let template_ids = parser.v9_template_ids();
1172 /// println!("Cached V9 templates: {:?}", template_ids);
1173 /// ```
1174 pub fn v9_template_ids(&self) -> Vec<u16> {
1175 let mut ids: Vec<u16> = self.v9_parser.templates.iter().map(|(id, _)| *id).collect();
1176 ids.extend(self.v9_parser.options_templates.iter().map(|(id, _)| *id));
1177 ids.sort_unstable();
1178 ids
1179 }
1180
1181 /// Lists all cached IPFIX template IDs.
1182 ///
1183 /// Note: This returns template IDs from both IPFIX and V9-format templates (IPFIX can contain both).
1184 ///
1185 /// # Examples
1186 ///
1187 /// ```rust
1188 /// use netflow_parser::NetflowParser;
1189 ///
1190 /// let parser = NetflowParser::default();
1191 /// let template_ids = parser.ipfix_template_ids();
1192 /// println!("Cached IPFIX templates: {:?}", template_ids);
1193 /// ```
1194 pub fn ipfix_template_ids(&self) -> Vec<u16> {
1195 let mut ids: Vec<u16> = self
1196 .ipfix_parser
1197 .templates
1198 .iter()
1199 .map(|(id, _)| *id)
1200 .collect();
1201 ids.extend(self.ipfix_parser.v9_templates.iter().map(|(id, _)| *id));
1202 ids.extend(
1203 self.ipfix_parser
1204 .ipfix_options_templates
1205 .iter()
1206 .map(|(id, _)| *id),
1207 );
1208 ids.extend(
1209 self.ipfix_parser
1210 .v9_options_templates
1211 .iter()
1212 .map(|(id, _)| *id),
1213 );
1214 ids.sort_unstable();
1215 ids
1216 }
1217
1218 /// Checks if a V9 template with the given ID is cached.
1219 ///
1220 /// Note: This uses `peek()` which does not affect LRU ordering.
1221 ///
1222 /// # Arguments
1223 ///
1224 /// * `template_id` - The template ID to check
1225 ///
1226 /// # Examples
1227 ///
1228 /// ```rust
1229 /// use netflow_parser::NetflowParser;
1230 ///
1231 /// let parser = NetflowParser::default();
1232 /// if parser.has_v9_template(256) {
1233 /// println!("Template 256 is cached");
1234 /// }
1235 /// ```
1236 pub fn has_v9_template(&self, template_id: u16) -> bool {
1237 self.v9_parser.templates.peek(&template_id).is_some()
1238 || self
1239 .v9_parser
1240 .options_templates
1241 .peek(&template_id)
1242 .is_some()
1243 }
1244
1245 /// Checks if an IPFIX template with the given ID is cached.
1246 ///
1247 /// Note: This uses `peek()` which does not affect LRU ordering.
1248 ///
1249 /// # Arguments
1250 ///
1251 /// * `template_id` - The template ID to check
1252 ///
1253 /// # Examples
1254 ///
1255 /// ```rust
1256 /// use netflow_parser::NetflowParser;
1257 ///
1258 /// let parser = NetflowParser::default();
1259 /// if parser.has_ipfix_template(256) {
1260 /// println!("Template 256 is cached");
1261 /// }
1262 /// ```
1263 pub fn has_ipfix_template(&self, template_id: u16) -> bool {
1264 self.ipfix_parser.templates.peek(&template_id).is_some()
1265 || self.ipfix_parser.v9_templates.peek(&template_id).is_some()
1266 || self
1267 .ipfix_parser
1268 .ipfix_options_templates
1269 .peek(&template_id)
1270 .is_some()
1271 || self
1272 .ipfix_parser
1273 .v9_options_templates
1274 .peek(&template_id)
1275 .is_some()
1276 }
1277
1278 /// Clears all cached V9 templates.
1279 ///
1280 /// This is useful for testing or when you need to force template re-learning.
1281 ///
1282 /// # Examples
1283 ///
1284 /// ```rust
1285 /// use netflow_parser::NetflowParser;
1286 ///
1287 /// let mut parser = NetflowParser::default();
1288 /// parser.clear_v9_templates();
1289 /// ```
1290 pub fn clear_v9_templates(&mut self) {
1291 self.v9_parser.templates.clear();
1292 self.v9_parser.options_templates.clear();
1293 }
1294
1295 /// Clears all cached IPFIX templates.
1296 ///
1297 /// This is useful for testing or when you need to force template re-learning.
1298 ///
1299 /// # Examples
1300 ///
1301 /// ```rust
1302 /// use netflow_parser::NetflowParser;
1303 ///
1304 /// let mut parser = NetflowParser::default();
1305 /// parser.clear_ipfix_templates();
1306 /// ```
1307 pub fn clear_ipfix_templates(&mut self) {
1308 self.ipfix_parser.templates.clear();
1309 self.ipfix_parser.v9_templates.clear();
1310 self.ipfix_parser.ipfix_options_templates.clear();
1311 self.ipfix_parser.v9_options_templates.clear();
1312 }
1313
1314 /// Takes a Netflow packet slice and returns a vector of Parsed Netflows.
1315 /// If we reach some parse error we return what items be have.
1316 ///
1317 /// # Examples
1318 ///
1319 /// ```rust
1320 /// use serde_json::json;
1321 /// use netflow_parser::NetflowParser;
1322 ///
1323 /// let v5_packet = [0, 5, 2, 0, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
1324 /// println!("{}", json!(NetflowParser::default().parse_bytes(&v5_packet)).to_string());
1325 /// ```
1326 ///
1327 /// ## Output:
1328 ///
1329 /// ```json
1330 /// [{"V5":{"header":{"count":1,"engine_id":7,"engine_type":6,"flow_sequence":33752069,"sampling_interval":2057,"sys_up_time":{"nanos":672000000,"secs":50332},"unix_nsecs":134807553,"unix_secs":83887623,"version":5},"sets":[{"d_octets":66051,"d_pkts":101124105,"dst_addr":"4.5.6.7","dst_as":515,"dst_mask":5,"dst_port":1029,"first":{"nanos":87000000,"secs":67438},"input":515,"last":{"nanos":553000000,"secs":134807},"next_hop":"8.9.0.1","output":1029,"pad1":6,"pad2":1543,"protocol_number":8,"protocol_type":"Egp","src_addr":"0.1.2.3","src_as":1,"src_mask":4,"src_port":515,"tcp_flags":7,"tos":9}]}}]
1331 /// ```
1332 ///
1333 #[inline]
1334 pub fn parse_bytes(&mut self, packet: &[u8]) -> Vec<NetflowPacket> {
1335 if packet.is_empty() {
1336 return vec![];
1337 }
1338
1339 let mut packets = Vec::new();
1340 let mut remaining = packet;
1341
1342 while !remaining.is_empty() {
1343 match self.parse_packet_by_version(remaining) {
1344 ParsedNetflow::Success {
1345 packet,
1346 remaining: new_remaining,
1347 } => {
1348 packets.push(packet);
1349 remaining = new_remaining;
1350 }
1351 ParsedNetflow::UnallowedVersion => {
1352 break;
1353 }
1354 ParsedNetflow::Error { error } => {
1355 // Only include first N bytes of remaining data in error to prevent memory exhaustion
1356 let remaining_sample = if remaining.len() > self.max_error_sample_size {
1357 remaining[..self.max_error_sample_size].to_vec()
1358 } else {
1359 remaining.to_vec()
1360 };
1361 packets.push(NetflowPacket::Error(NetflowPacketError {
1362 error,
1363 remaining: remaining_sample,
1364 }));
1365 break;
1366 }
1367 }
1368 }
1369
1370 packets
1371 }
1372
1373 /// Returns an iterator that yields NetflowPacket items without allocating a Vec.
1374 /// This is useful for processing large batches of packets without collecting all results in memory.
1375 ///
1376 /// # Examples
1377 ///
1378 /// ```rust
1379 /// use netflow_parser::{NetflowParser, NetflowPacket};
1380 ///
1381 /// let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
1382 /// let mut parser = NetflowParser::default();
1383 ///
1384 /// for packet in parser.iter_packets(&v5_packet) {
1385 /// match packet {
1386 /// NetflowPacket::V5(v5) => println!("V5 packet: {:?}", v5.header.version),
1387 /// NetflowPacket::Error(e) => println!("Error: {:?}", e),
1388 /// _ => (),
1389 /// }
1390 /// }
1391 /// ```
1392 #[inline]
1393 pub fn iter_packets<'a>(&'a mut self, packet: &'a [u8]) -> NetflowPacketIterator<'a> {
1394 NetflowPacketIterator {
1395 parser: self,
1396 remaining: packet,
1397 errored: false,
1398 }
1399 }
1400
1401 #[inline]
1402 fn parse_packet_by_version<'a>(&mut self, packet: &'a [u8]) -> ParsedNetflow<'a> {
1403 match GenericNetflowHeader::parse(packet) {
1404 Ok((packet, header)) if self.allowed_versions.contains(&header.version) => {
1405 match header.version {
1406 5 => V5Parser::parse(packet),
1407 7 => V7Parser::parse(packet),
1408 9 => self.v9_parser.parse(packet),
1409 10 => self.ipfix_parser.parse(packet),
1410 _ => ParsedNetflow::Error {
1411 error: NetflowParseError::UnknownVersion(packet.to_vec()),
1412 },
1413 }
1414 }
1415 Ok((_, _)) => ParsedNetflow::UnallowedVersion,
1416 Err(e) => ParsedNetflow::Error {
1417 error: NetflowParseError::Incomplete(e.to_string()),
1418 },
1419 }
1420 }
1421
1422 /// Takes a Netflow packet slice and returns a vector of Parsed NetflowCommonFlowSet
1423 #[cfg(feature = "netflow_common")]
1424 #[inline]
1425 pub fn parse_bytes_as_netflow_common_flowsets(
1426 &mut self,
1427 packet: &[u8],
1428 ) -> Vec<NetflowCommonFlowSet> {
1429 let netflow_packets = self.parse_bytes(packet);
1430 netflow_packets
1431 .iter()
1432 .flat_map(|n| n.as_netflow_common().unwrap_or_default().flowsets)
1433 .collect()
1434 }
1435}