netflow_parser/lib.rs
1//! # netflow_parser
2//!
3//! A Netflow Parser library for Cisco V5, V7, V9, and IPFIX written in Rust. Supports chaining of multiple versions in the same stream.
4//!
5//! > **⚠️ Breaking Changes in v0.7.0:** The Template TTL API has been simplified to only support time-based expiration.
6//! > Packet-based and combined TTL modes have been removed. See the [RELEASES.md](https://github.com/mikemiles-dev/netflow_parser/blob/main/RELEASES.md)
7//! > for the full migration guide.
8//!
9//! ## Quick Start
10//!
11//! ### Using the Builder Pattern (Recommended)
12//!
13//! ```rust
14//! use netflow_parser::NetflowParser;
15//! use netflow_parser::variable_versions::ttl::TtlConfig;
16//! use std::time::Duration;
17//!
18//! // Create a parser with custom configuration
19//! let mut parser = NetflowParser::builder()
20//! .with_cache_size(2000)
21//! .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
22//! .build()
23//! .expect("Failed to build parser");
24//!
25//! // Parse packets
26//! let buffer = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7];
27//! let packets = parser.parse_bytes(&buffer);
28//! ```
29//!
30//! ### Using Default Configuration
31//!
32//! ```rust
33//! use netflow_parser::{NetflowParser, NetflowPacket};
34//!
35//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
36//! match NetflowParser::default().parse_bytes(&v5_packet).first() {
37//! Some(NetflowPacket::V5(v5)) => assert_eq!(v5.header.version, 5),
38//! Some(NetflowPacket::Error(e)) => println!("{:?}", e),
39//! _ => (),
40//! }
41//! ```
42//!
43//! ## Want Serialization such as JSON?
44//! Structures fully support serialization. Below is an example using the serde_json macro:
45//! ```rust
46//! use serde_json::json;
47//! use netflow_parser::NetflowParser;
48//!
49//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
50//! println!("{}", json!(NetflowParser::default().parse_bytes(&v5_packet)).to_string());
51//! ```
52//!
53//! ```json
54//! [
55//! {
56//! "V5": {
57//! "header": {
58//! "count": 1,
59//! "engine_id": 7,
60//! "engine_type": 6,
61//! "flow_sequence": 33752069,
62//! "sampling_interval": 2057,
63//! "sys_up_time": { "nanos": 672000000, "secs": 50332 },
64//! "unix_nsecs": 134807553,
65//! "unix_secs": 83887623,
66//! "version": 5
67//! },
68//! "sets": [
69//! {
70//! "d_octets": 66051,
71//! "d_pkts": 101124105,
72//! "dst_addr": "4.5.6.7",
73//! "dst_as": 515,
74//! "dst_mask": 5,
75//! "dst_port": 1029,
76//! "first": { "nanos": 87000000, "secs": 67438 },
77//! "input": 515,
78//! "last": { "nanos": 553000000, "secs": 134807 },
79//! "next_hop": "8.9.0.1",
80//! "output": 1029,
81//! "pad1": 6,
82//! "pad2": 1543,
83//! "protocol_number": 8,
84//! "protocol_type": "Egp",
85//! "src_addr": "0.1.2.3",
86//! "src_as": 1,
87//! "src_mask": 4,
88//! "src_port": 515,
89//! "tcp_flags": 7,
90//! "tos": 9
91//! }
92//! ]
93//! }
94//! }
95//! ]
96//! ```
97//!
98//! ## Filtering for a Specific Version
99//!
100//! ```rust
101//! use netflow_parser::{NetflowParser, NetflowPacket};
102//!
103//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
104//! let parsed = NetflowParser::default().parse_bytes(&v5_packet);
105//!
106//! let v5_parsed: Vec<NetflowPacket> = parsed.into_iter().filter(|p| p.is_v5()).collect();
107//! ```
108//!
109//! ## Iterator API
110//!
111//! For high-performance scenarios where you want to avoid allocating a `Vec`, you can use the iterator API to process packets one-by-one as they're parsed:
112//!
113//! ```rust
114//! use netflow_parser::{NetflowParser, NetflowPacket};
115//!
116//! # let buffer = [0u8; 72];
117//! let mut parser = NetflowParser::default();
118//!
119//! // Process packets without collecting into a Vec
120//! for packet in parser.iter_packets(&buffer) {
121//! match packet {
122//! NetflowPacket::V5(v5) => {
123//! // Process V5 packet
124//! println!("V5 packet from {}", v5.header.version);
125//! }
126//! NetflowPacket::V9(v9) => {
127//! // Process V9 packet
128//! for flowset in &v9.flowsets {
129//! // Handle flowsets
130//! }
131//! }
132//! NetflowPacket::IPFix(ipfix) => {
133//! // Process IPFIX packet
134//! }
135//! NetflowPacket::Error(e) => {
136//! eprintln!("Parse error: {:?}", e);
137//! }
138//! _ => {}
139//! }
140//! }
141//! ```
142//!
143//! The iterator provides access to unconsumed bytes for advanced use cases:
144//!
145//! ```rust
146//! use netflow_parser::NetflowParser;
147//!
148//! # let buffer = [0u8; 72];
149//! let mut parser = NetflowParser::default();
150//! let mut iter = parser.iter_packets(&buffer);
151//!
152//! while let Some(packet) = iter.next() {
153//! // Process packet
154//! # _ = packet;
155//! }
156//!
157//! // Check if all bytes were consumed
158//! if !iter.is_complete() {
159//! println!("Warning: {} bytes remain unconsumed", iter.remaining().len());
160//! }
161//! ```
162//!
163//! ### Benefits of Iterator API
164//!
165//! - **Zero allocation**: Packets are yielded one-by-one without allocating a `Vec`
166//! - **Memory efficient**: Ideal for processing large batches or continuous streams
167//! - **Lazy evaluation**: Only parses packets as you consume them
168//! - **Template caching preserved**: V9/IPFIX template state is maintained across iterations
169//! - **Composable**: Works with standard Rust iterator methods (`.filter()`, `.map()`, `.take()`, etc.)
170//! - **Buffer inspection**: Access unconsumed bytes via `.remaining()` and check completion with `.is_complete()`
171//!
172//! ### Iterator Examples
173//!
174//! ```rust
175//! # use netflow_parser::{NetflowParser, NetflowPacket};
176//! # let buffer = [0u8; 72];
177//! # let mut parser = NetflowParser::default();
178//! // Count V5 packets without collecting
179//! let count = parser.iter_packets(&buffer)
180//! .filter(|p| p.is_v5())
181//! .count();
182//!
183//! // Process only the first 10 packets
184//! for packet in parser.iter_packets(&buffer).take(10) {
185//! // Handle packet
186//! # _ = packet;
187//! }
188//!
189//! // Collect only if needed (equivalent to parse_bytes())
190//! let packets: Vec<_> = parser.iter_packets(&buffer).collect();
191//!
192//! // Check unconsumed bytes (useful for mixed protocol streams)
193//! let mut iter = parser.iter_packets(&buffer);
194//! for packet in &mut iter {
195//! // Process packet
196//! # _ = packet;
197//! }
198//! if !iter.is_complete() {
199//! let remaining = iter.remaining();
200//! // Handle non-netflow data at end of buffer
201//! # _ = remaining;
202//! }
203//! ```
204//!
205//! ## Parsing Out Unneeded Versions
206//! If you only care about a specific version or versions you can specify `allowed_versions`:
207//! ```rust
208//! use netflow_parser::{NetflowParser, NetflowPacket};
209//!
210//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
211//! let mut parser = NetflowParser::default();
212//! parser.allowed_versions = [7, 9].into();
213//! let parsed = parser.parse_bytes(&v5_packet);
214//! ```
215//!
216//! This code will return an empty Vec as version 5 is not allowed.
217//!
218//! ## Error Handling Configuration
219//!
220//! To prevent memory exhaustion from malformed packets, the parser limits the size of error buffer samples. By default, only the first 256 bytes of unparseable data are stored in error messages. You can customize this limit for all parsers:
221//!
222//! ```rust
223//! use netflow_parser::NetflowParser;
224//!
225//! let mut parser = NetflowParser::default();
226//!
227//! // Configure maximum error buffer size for the main parser (default: 256 bytes)
228//! // This applies to generic parsing errors
229//! parser.max_error_sample_size = 512;
230//!
231//! // Configure maximum error buffer size for V9 (default: 256 bytes)
232//! parser.v9_parser.max_error_sample_size = 512;
233//!
234//! // Configure maximum error buffer size for IPFIX (default: 256 bytes)
235//! parser.ipfix_parser.max_error_sample_size = 512;
236//!
237//! # let some_packet = [0u8; 72];
238//! let parsed = parser.parse_bytes(&some_packet);
239//! ```
240//!
241//! This setting helps prevent memory exhaustion when processing malformed or malicious packets while still providing enough context for debugging.
242//!
243//! ## Netflow Common
244//!
245//! We have included a `NetflowCommon` and `NetflowCommonFlowSet` structure.
246//! This will allow you to use common fields without unpacking values from specific versions.
247//! If the packet flow does not have the matching field it will simply be left as `None`.
248//!
249//! ### NetflowCommon and NetflowCommonFlowSet Struct:
250//! ```rust
251//! use std::net::IpAddr;
252//! use netflow_parser::protocol::ProtocolTypes;
253//!
254//! #[derive(Debug, Default)]
255//! pub struct NetflowCommon {
256//! pub version: u16,
257//! pub timestamp: u32,
258//! pub flowsets: Vec<NetflowCommonFlowSet>,
259//! }
260//!
261//! #[derive(Debug, Default)]
262//! struct NetflowCommonFlowSet {
263//! src_addr: Option<IpAddr>,
264//! dst_addr: Option<IpAddr>,
265//! src_port: Option<u16>,
266//! dst_port: Option<u16>,
267//! protocol_number: Option<u8>,
268//! protocol_type: Option<ProtocolTypes>,
269//! first_seen: Option<u32>,
270//! last_seen: Option<u32>,
271//! src_mac: Option<String>,
272//! dst_mac: Option<String>,
273//! }
274//! ```
275//!
276//! ### Converting NetflowPacket to NetflowCommon
277//!
278//! ```rust,ignore
279//! use netflow_parser::{NetflowParser, NetflowPacket};
280//!
281//! let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3,
282//! 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1,
283//! 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7];
284//! let netflow_common = NetflowParser::default()
285//! .parse_bytes(&v5_packet)
286//! .first()
287//! .unwrap()
288//! .as_netflow_common()
289//! .unwrap();
290//!
291//! for common_flow in netflow_common.flowsets.iter() {
292//! println!("Src Addr: {} Dst Addr: {}", common_flow.src_addr.unwrap(), common_flow.dst_addr.unwrap());
293//! }
294//! ```
295//!
296//! ### Flattened flowsets
297//!
298//! To gather all flowsets from all packets into a flattened vector:
299//!
300//! ```rust,ignore
301//! use netflow_parser::NetflowParser;
302//!
303//! # let v5_packet = [0u8; 72];
304//! let flowsets = NetflowParser::default().parse_bytes_as_netflow_common_flowsets(&v5_packet);
305//! ```
306//!
307//! ### Custom Field Mappings for V9 and IPFIX
308//!
309//! By default, NetflowCommon maps standard IANA fields to the common structure. However, you can customize which fields are used for V9 and IPFIX packets using configuration structs. This is useful when:
310//!
311//! - You want to prefer IPv6 addresses over IPv4
312//! - Your vendor uses non-standard field mappings
313//! - You need to extract data from vendor-specific enterprise fields
314//!
315//! #### V9 Custom Field Mapping
316//!
317//! ```rust,ignore
318//! use netflow_parser::netflow_common::{NetflowCommon, V9FieldMappingConfig};
319//! use netflow_parser::variable_versions::v9_lookup::V9Field;
320//!
321//! // Create a custom configuration that prefers IPv6 addresses
322//! let mut config = V9FieldMappingConfig::default();
323//! config.src_addr.primary = V9Field::Ipv6SrcAddr;
324//! config.src_addr.fallback = Some(V9Field::Ipv4SrcAddr);
325//! config.dst_addr.primary = V9Field::Ipv6DstAddr;
326//! config.dst_addr.fallback = Some(V9Field::Ipv4DstAddr);
327//!
328//! // Use with a parsed V9 packet
329//! // let common = NetflowCommon::from_v9_with_config(&v9_packet, &config);
330//! ```
331//!
332//! #### IPFIX Custom Field Mapping
333//!
334//! ```rust,ignore
335//! use netflow_parser::netflow_common::{NetflowCommon, IPFixFieldMappingConfig};
336//! use netflow_parser::variable_versions::ipfix_lookup::{IPFixField, IANAIPFixField};
337//!
338//! // Create a custom configuration that prefers IPv6 addresses
339//! let mut config = IPFixFieldMappingConfig::default();
340//! config.src_addr.primary = IPFixField::IANA(IANAIPFixField::SourceIpv6address);
341//! config.src_addr.fallback = Some(IPFixField::IANA(IANAIPFixField::SourceIpv4address));
342//! config.dst_addr.primary = IPFixField::IANA(IANAIPFixField::DestinationIpv6address);
343//! config.dst_addr.fallback = Some(IPFixField::IANA(IANAIPFixField::DestinationIpv4address));
344//!
345//! // Use with a parsed IPFIX packet
346//! // let common = NetflowCommon::from_ipfix_with_config(&ipfix_packet, &config);
347//! ```
348//!
349//! #### Available Configuration Fields
350//!
351//! Both `V9FieldMappingConfig` and `IPFixFieldMappingConfig` support configuring:
352//!
353//! | Field | Description | Default V9 Field | Default IPFIX Field |
354//! |-------|-------------|------------------|---------------------|
355//! | `src_addr` | Source IP address | Ipv4SrcAddr (fallback: Ipv6SrcAddr) | SourceIpv4address (fallback: SourceIpv6address) |
356//! | `dst_addr` | Destination IP address | Ipv4DstAddr (fallback: Ipv6DstAddr) | DestinationIpv4address (fallback: DestinationIpv6address) |
357//! | `src_port` | Source port | L4SrcPort | SourceTransportPort |
358//! | `dst_port` | Destination port | L4DstPort | DestinationTransportPort |
359//! | `protocol` | Protocol number | Protocol | ProtocolIdentifier |
360//! | `first_seen` | Flow start time | FirstSwitched | FlowStartSysUpTime |
361//! | `last_seen` | Flow end time | LastSwitched | FlowEndSysUpTime |
362//! | `src_mac` | Source MAC address | InSrcMac | SourceMacaddress |
363//! | `dst_mac` | Destination MAC address | InDstMac | DestinationMacaddress |
364//!
365//! Each field mapping has a `primary` field (always checked first) and an optional `fallback` field (used if primary is not present in the flow record).
366//!
367//! ## Re-Exporting Flows
368//!
369//! Parsed V5, V7, V9, and IPFIX packets can be re-exported back into bytes.
370//!
371//! **V9/IPFIX Padding Behavior:**
372//! - For **parsed packets**: Original padding is preserved exactly for byte-perfect round-trips
373//! - For **manually created packets**: Padding is automatically calculated to align FlowSets to 4-byte boundaries
374//!
375//! **Creating Data Structs:**
376//! For convenience, use `Data::new()` and `OptionsData::new()` to create data structures without manually specifying padding:
377//!
378//! ```rust,ignore
379//! use netflow_parser::variable_versions::ipfix::Data;
380//!
381//! // Padding is automatically set to empty vec and calculated during export
382//! let data = Data::new(vec![vec![
383//! (field1, value1),
384//! (field2, value2),
385//! ]]);
386//! ```
387//!
388//! See `examples/manual_ipfix_creation.rs` for a complete example of creating IPFIX packets from scratch.
389//!
390//! ```rust
391//! use netflow_parser::{NetflowParser, NetflowPacket};
392//!
393//! let packet = [
394//! 0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3,
395//! 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1,
396//! 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,
397//! ];
398//! if let NetflowPacket::V5(v5) = NetflowParser::default()
399//! .parse_bytes(&packet)
400//! .first()
401//! .unwrap()
402//! {
403//! assert_eq!(v5.to_be_bytes(), packet);
404//! }
405//! ```
406//!
407//! ## Template Cache Configuration
408//!
409//! V9 and IPFIX parsers use LRU (Least Recently Used) caching to store templates with a configurable size limit. This prevents memory exhaustion from template flooding attacks while maintaining good performance for legitimate traffic.
410//!
411//! ### Default Behavior
412//!
413//! By default, parsers cache up to 1000 templates:
414//!
415//! ```rust
416//! use netflow_parser::NetflowParser;
417//!
418//! // Uses default cache size of 1000 templates per parser
419//! let parser = NetflowParser::default();
420//! ```
421//!
422//! ### Custom Cache Size
423//!
424//! Use the builder pattern to configure cache sizes:
425//!
426//! ```rust
427//! use netflow_parser::NetflowParser;
428//!
429//! // Configure both V9 and IPFIX with the same cache size
430//! let parser = NetflowParser::builder()
431//! .with_cache_size(2000)
432//! .build()
433//! .expect("Failed to build parser");
434//!
435//! // Configure V9 and IPFIX independently
436//! let parser = NetflowParser::builder()
437//! .with_v9_cache_size(1000)
438//! .with_ipfix_cache_size(5000)
439//! .build()
440//! .expect("Failed to build parser");
441//! ```
442//!
443//! ### Cache Behavior
444//!
445//! - When the cache is full, the least recently used template is evicted
446//! - Templates are keyed by template ID (per source)
447//! - Each parser instance maintains its own template cache
448//! - For multi-source deployments, create separate parser instances per source
449//!
450//! ### Template TTL (Time-to-Live)
451//!
452//! Optionally configure templates to expire after a time duration. This is useful for:
453//! - Handling exporters that reuse template IDs with different schemas
454//! - Forcing periodic template refresh from exporters
455//! - Testing template re-learning behavior
456//!
457//! **Note:** TTL is disabled by default. Templates persist until LRU eviction unless explicitly configured.
458//!
459//! #### Configuration Examples
460//!
461//! ```rust
462//! use netflow_parser::NetflowParser;
463//! use netflow_parser::variable_versions::ttl::TtlConfig;
464//! use std::time::Duration;
465//!
466//! // Templates expire after 2 hours
467//! let parser = NetflowParser::builder()
468//! .with_cache_size(1000)
469//! .with_ttl(TtlConfig::new(Duration::from_secs(2 * 3600)))
470//! .build()
471//! .unwrap();
472//!
473//! // Using default TTL (2 hours)
474//! let parser = NetflowParser::builder()
475//! .with_cache_size(1000)
476//! .with_ttl(TtlConfig::default())
477//! .build()
478//! .unwrap();
479//!
480//! // Different TTL for V9 and IPFIX
481//! let parser = NetflowParser::builder()
482//! .with_v9_ttl(TtlConfig::new(Duration::from_secs(3600)))
483//! .with_ipfix_ttl(TtlConfig::new(Duration::from_secs(2 * 3600)))
484//! .build()
485//! .unwrap();
486//! ```
487//!
488//! ## V9/IPFIX Notes
489//!
490//! Parse the data (`&[u8]`) like any other version. The parser (`NetflowParser`) caches parsed templates using LRU eviction, so you can send header/data flowset combos and it will use the cached templates. Templates are automatically cached and evicted when the cache limit is reached.
491//!
492//! **Template Cache Introspection:**
493//! Use the introspection methods to inspect template cache state without affecting LRU ordering:
494//!
495//! ```rust
496//! use netflow_parser::NetflowParser;
497//! let parser = NetflowParser::default();
498//!
499//! // Get cache statistics
500//! let stats = parser.v9_cache_stats();
501//! println!("V9 cache: {}/{} templates", stats.current_size, stats.max_size);
502//!
503//! // List all cached template IDs
504//! let template_ids = parser.v9_template_ids();
505//! println!("Cached templates: {:?}", template_ids);
506//!
507//! // Check if a specific template exists (doesn't affect LRU)
508//! if parser.has_v9_template(256) {
509//! println!("Template 256 is cached");
510//! }
511//! ```
512//!
513//! **IPFIX Note:** We only parse sequence number and domain id, it is up to you if you wish to validate it.
514//!
515//! To access templates flowset of a processed V9/IPFix flowset you can find the `flowsets` attribute on the Parsed Record. In there you can find `Templates`, `Option Templates`, and `Data` Flowsets.
516//!
517//! ## Performance & Thread Safety
518//!
519//! ### Thread Safety
520//!
521//! Parsers (`NetflowParser`, `V9Parser`, `IPFixParser`) are **not thread-safe** and should not be shared across threads without external synchronization. Each parser maintains internal state (template caches) that is modified during parsing.
522//!
523//! **Recommended pattern for multi-threaded applications:**
524//! - Create one parser instance per thread
525//! - Each thread processes packets from a single router/source
526//! - See `examples/netflow_udp_listener_multi_threaded.rs` for implementation example
527//!
528//! ### Performance Optimizations
529//!
530//! This library includes several performance optimizations:
531//!
532//! 1. **Single-pass field caching** - NetflowCommon conversions use efficient single-pass lookups
533//! 2. **Minimal cloning** - Template storage avoids unnecessary vector clones
534//! 3. **Optimized string processing** - Single-pass filtering and prefix stripping
535//! 4. **Capacity pre-allocation** - Vectors pre-allocate when sizes are known
536//! 5. **Bounded error buffers** - Error handling limits memory consumption to prevent exhaustion
537//!
538//! **Best practices for optimal performance:**
539//! - Reuse parser instances instead of creating new ones for each packet
540//! - Use `iter_packets()` instead of `parse_bytes()` when you don't need all packets in a Vec
541//! - Use `parse_bytes_as_netflow_common_flowsets()` when you only need flow data
542//! - For V9/IPFIX, batch process packets from the same source to maximize template cache hits
543//!
544//! ## Features
545//!
546//! * `parse_unknown_fields` - When enabled fields not listed in this library will attempt to be parsed as a Vec of bytes and the field_number listed. When disabled an error is thrown when attempting to parse those fields. Enabled by default.
547//! * `netflow_common` - When enabled provides `NetflowCommon` and `NetflowCommonFlowSet` structures for working with common fields across different Netflow versions. Disabled by default.
548//!
549//! ## Included Examples
550//!
551//! Examples have been included mainly for those who want to use this parser to read from a Socket and parse netflow. In those cases with V9/IPFix it is best to create a new parser for each router. There are both single threaded and multi-threaded examples in the examples directory.
552//!
553//! Examples that listen on a specific port use 9995 by default, however netflow can be configurated to use a variety of URP ports:
554//! * **2055**: The most widely recognized default for NetFlow.
555//! * **9995 / 9996**: Popular alternatives, especially with Cisco devices.
556//! * **9025, 9026**: Other recognized port options.
557//! * **6343**: The default for sFlow, often used alongside NetFlow.
558//! * **4739**: The default port for IPFIX (a NetFlow successor).
559//!
560//! To run:
561//!
562//! ```cargo run --example netflow_udp_listener_multi_threaded```
563//!
564//! ```cargo run --example netflow_udp_listener_single_threaded```
565//!
566//! ```cargo run --example netflow_udp_listener_tokio```
567//!
568//! ```cargo run --example netflow_pcap```
569//!
570//! ```cargo run --example manual_ipfix_creation```
571//!
572//! The pcap example also shows how to cache flows that have not yet discovered a template.
573//!
574//! ## Support My Work
575//!
576//! If you find my work helpful, consider supporting me!
577//!
578//! [](https://ko-fi.com/michaelmileusnich)
579//!
580//! [](https://github.com/sponsors/mikemiles-dev)
581
582#[cfg(feature = "netflow_common")]
583pub mod netflow_common;
584pub mod protocol;
585pub mod static_versions;
586mod tests;
587pub mod variable_versions;
588
589#[cfg(feature = "netflow_common")]
590use crate::netflow_common::{NetflowCommon, NetflowCommonError, NetflowCommonFlowSet};
591
592use static_versions::{
593 v5::{V5, V5Parser},
594 v7::{V7, V7Parser},
595};
596use variable_versions::Config;
597use variable_versions::ipfix::{IPFix, IPFixParser};
598use variable_versions::v9::{V9, V9Parser};
599
600use nom_derive::{Nom, Parse};
601use serde::Serialize;
602
603use std::collections::HashSet;
604
605/// Enum of supported Netflow Versions
606#[derive(Debug, Clone, Serialize)]
607pub enum NetflowPacket {
608 /// Version 5
609 V5(V5),
610 /// Version 7
611 V7(V7),
612 /// Version 9
613 V9(V9),
614 /// IPFix
615 IPFix(IPFix),
616 /// Error
617 Error(NetflowPacketError),
618}
619
620impl NetflowPacket {
621 pub fn is_v5(&self) -> bool {
622 matches!(self, Self::V5(_v))
623 }
624 pub fn is_v7(&self) -> bool {
625 matches!(self, Self::V7(_v))
626 }
627 pub fn is_v9(&self) -> bool {
628 matches!(self, Self::V9(_v))
629 }
630 pub fn is_ipfix(&self) -> bool {
631 matches!(self, Self::IPFix(_v))
632 }
633 pub fn is_error(&self) -> bool {
634 matches!(self, Self::Error(_v))
635 }
636 #[cfg(feature = "netflow_common")]
637 pub fn as_netflow_common(&self) -> Result<NetflowCommon, NetflowCommonError> {
638 self.try_into()
639 }
640}
641
642#[derive(Nom)]
643/// Generic Netflow Header for shared versions
644struct GenericNetflowHeader {
645 version: u16,
646}
647
648/// Main parser for Netflow packets supporting V5, V7, V9, and IPFIX.
649///
650/// Use [`NetflowParser::builder()`] for ergonomic configuration with the builder pattern,
651/// or [`NetflowParser::default()`] for quick setup with defaults.
652///
653/// # Examples
654///
655/// ```rust
656/// use netflow_parser::NetflowParser;
657/// use netflow_parser::variable_versions::ttl::TtlConfig;
658/// use std::time::Duration;
659///
660/// // Using builder pattern (recommended)
661/// let parser = NetflowParser::builder()
662/// .with_cache_size(2000)
663/// .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
664/// .build()
665/// .expect("Failed to build parser");
666///
667/// // Using default
668/// let parser = NetflowParser::default();
669/// ```
670#[derive(Debug)]
671pub struct NetflowParser {
672 pub v9_parser: V9Parser,
673 pub ipfix_parser: IPFixParser,
674 pub allowed_versions: HashSet<u16>,
675 /// Maximum number of bytes to include in error samples to prevent memory exhaustion.
676 /// Defaults to 256 bytes.
677 pub max_error_sample_size: usize,
678}
679
680/// Statistics about template cache utilization.
681#[derive(Debug, Clone)]
682pub struct CacheStats {
683 /// Current number of cached templates
684 pub current_size: usize,
685 /// Maximum cache size before LRU eviction
686 pub max_size: usize,
687 /// TTL configuration (if enabled)
688 pub ttl_config: Option<variable_versions::ttl::TtlConfig>,
689}
690
691/// Builder for configuring and constructing a [`NetflowParser`].
692///
693/// # Examples
694///
695/// ```rust
696/// use netflow_parser::NetflowParser;
697/// use netflow_parser::variable_versions::ttl::TtlConfig;
698/// use std::collections::HashSet;
699/// use std::time::Duration;
700///
701/// let parser = NetflowParser::builder()
702/// .with_cache_size(2000)
703/// .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
704/// .with_allowed_versions([5, 9, 10].into())
705/// .with_max_error_sample_size(512)
706/// .build()
707/// .expect("Failed to build parser");
708/// ```
709#[derive(Debug, Clone)]
710pub struct NetflowParserBuilder {
711 v9_config: Config,
712 ipfix_config: Config,
713 allowed_versions: HashSet<u16>,
714 max_error_sample_size: usize,
715}
716
717impl Default for NetflowParserBuilder {
718 fn default() -> Self {
719 Self {
720 v9_config: Config::new(1000, None),
721 ipfix_config: Config::new(1000, None),
722 allowed_versions: [5, 7, 9, 10].iter().cloned().collect(),
723 max_error_sample_size: 256,
724 }
725 }
726}
727
728impl NetflowParserBuilder {
729 /// Sets the template cache size for both V9 and IPFIX parsers.
730 ///
731 /// # Arguments
732 ///
733 /// * `size` - Maximum number of templates to cache (must be > 0)
734 ///
735 /// # Examples
736 ///
737 /// ```rust
738 /// use netflow_parser::NetflowParser;
739 ///
740 /// let parser = NetflowParser::builder()
741 /// .with_cache_size(2000)
742 /// .build()
743 /// .expect("Failed to build parser");
744 /// ```
745 pub fn with_cache_size(mut self, size: usize) -> Self {
746 self.v9_config.max_template_cache_size = size;
747 self.ipfix_config.max_template_cache_size = size;
748 self
749 }
750
751 /// Sets the V9 parser template cache size independently.
752 ///
753 /// # Arguments
754 ///
755 /// * `size` - Maximum number of templates to cache (must be > 0)
756 pub fn with_v9_cache_size(mut self, size: usize) -> Self {
757 self.v9_config.max_template_cache_size = size;
758 self
759 }
760
761 /// Sets the IPFIX parser template cache size independently.
762 ///
763 /// * `size` - Maximum number of templates to cache (must be > 0)
764 pub fn with_ipfix_cache_size(mut self, size: usize) -> Self {
765 self.ipfix_config.max_template_cache_size = size;
766 self
767 }
768
769 /// Sets the TTL configuration for both V9 and IPFIX parsers.
770 ///
771 /// # Arguments
772 ///
773 /// * `ttl` - TTL configuration (time-based)
774 ///
775 /// # Examples
776 ///
777 /// ```rust
778 /// use netflow_parser::NetflowParser;
779 /// use netflow_parser::variable_versions::ttl::TtlConfig;
780 /// use std::time::Duration;
781 ///
782 /// let parser = NetflowParser::builder()
783 /// .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
784 /// .build()
785 /// .expect("Failed to build parser");
786 /// ```
787 pub fn with_ttl(mut self, ttl: variable_versions::ttl::TtlConfig) -> Self {
788 self.v9_config.ttl_config = Some(ttl.clone());
789 self.ipfix_config.ttl_config = Some(ttl);
790 self
791 }
792
793 /// Sets the TTL configuration for V9 parser independently.
794 pub fn with_v9_ttl(mut self, ttl: variable_versions::ttl::TtlConfig) -> Self {
795 self.v9_config.ttl_config = Some(ttl);
796 self
797 }
798
799 /// Sets the TTL configuration for IPFIX parser independently.
800 pub fn with_ipfix_ttl(mut self, ttl: variable_versions::ttl::TtlConfig) -> Self {
801 self.ipfix_config.ttl_config = Some(ttl);
802 self
803 }
804
805 /// Sets which Netflow versions are allowed to be parsed.
806 ///
807 /// # Arguments
808 ///
809 /// * `versions` - Set of allowed version numbers (5, 7, 9, 10)
810 ///
811 /// # Examples
812 ///
813 /// ```rust
814 /// use netflow_parser::NetflowParser;
815 ///
816 /// // Only parse V9 and IPFIX
817 /// let parser = NetflowParser::builder()
818 /// .with_allowed_versions([9, 10].into())
819 /// .build()
820 /// .expect("Failed to build parser");
821 /// ```
822 pub fn with_allowed_versions(mut self, versions: HashSet<u16>) -> Self {
823 self.allowed_versions = versions;
824 self
825 }
826
827 /// Sets the maximum error sample size for error reporting.
828 ///
829 /// # Arguments
830 ///
831 /// * `size` - Maximum bytes to include in error messages
832 ///
833 /// # Examples
834 ///
835 /// ```rust
836 /// use netflow_parser::NetflowParser;
837 ///
838 /// let parser = NetflowParser::builder()
839 /// .with_max_error_sample_size(512)
840 /// .build()
841 /// .expect("Failed to build parser");
842 /// ```
843 pub fn with_max_error_sample_size(mut self, size: usize) -> Self {
844 self.max_error_sample_size = size;
845 self
846 }
847
848 /// Registers a custom enterprise field definition for both V9 and IPFIX parsers.
849 ///
850 /// This allows library users to define their own enterprise-specific fields without
851 /// modifying the library source code. Registered fields will be parsed according to
852 /// their specified data type instead of falling back to raw bytes.
853 ///
854 /// # Arguments
855 ///
856 /// * `def` - Enterprise field definition containing enterprise number, field number, name, and data type
857 ///
858 /// # Examples
859 ///
860 /// ```rust
861 /// use netflow_parser::NetflowParser;
862 /// use netflow_parser::variable_versions::enterprise_registry::EnterpriseFieldDef;
863 /// use netflow_parser::variable_versions::data_number::FieldDataType;
864 ///
865 /// let parser = NetflowParser::builder()
866 /// .register_enterprise_field(EnterpriseFieldDef::new(
867 /// 12345, // Enterprise number
868 /// 1, // Field number
869 /// "customMetric",
870 /// FieldDataType::UnsignedDataNumber,
871 /// ))
872 /// .register_enterprise_field(EnterpriseFieldDef::new(
873 /// 12345,
874 /// 2,
875 /// "customApplicationName",
876 /// FieldDataType::String,
877 /// ))
878 /// .build()
879 /// .expect("Failed to build parser");
880 /// ```
881 pub fn register_enterprise_field(
882 mut self,
883 def: variable_versions::enterprise_registry::EnterpriseFieldDef,
884 ) -> Self {
885 self.v9_config.enterprise_registry.register(def.clone());
886 self.ipfix_config.enterprise_registry.register(def);
887 self
888 }
889
890 /// Registers multiple custom enterprise field definitions at once.
891 ///
892 /// # Arguments
893 ///
894 /// * `defs` - Iterator of enterprise field definitions
895 ///
896 /// # Examples
897 ///
898 /// ```rust
899 /// use netflow_parser::NetflowParser;
900 /// use netflow_parser::variable_versions::enterprise_registry::EnterpriseFieldDef;
901 /// use netflow_parser::variable_versions::data_number::FieldDataType;
902 ///
903 /// let fields = vec![
904 /// EnterpriseFieldDef::new(12345, 1, "field1", FieldDataType::UnsignedDataNumber),
905 /// EnterpriseFieldDef::new(12345, 2, "field2", FieldDataType::String),
906 /// EnterpriseFieldDef::new(12345, 3, "field3", FieldDataType::Ip4Addr),
907 /// ];
908 ///
909 /// let parser = NetflowParser::builder()
910 /// .register_enterprise_fields(fields)
911 /// .build()
912 /// .expect("Failed to build parser");
913 /// ```
914 pub fn register_enterprise_fields(
915 mut self,
916 defs: impl IntoIterator<Item = variable_versions::enterprise_registry::EnterpriseFieldDef>,
917 ) -> Self {
918 for def in defs {
919 self.v9_config.enterprise_registry.register(def.clone());
920 self.ipfix_config.enterprise_registry.register(def);
921 }
922 self
923 }
924
925 /// Builds the configured [`NetflowParser`].
926 ///
927 /// # Errors
928 ///
929 /// Returns an error if:
930 /// - Template cache size is 0
931 /// - Parser initialization fails
932 ///
933 /// # Examples
934 ///
935 /// ```rust
936 /// use netflow_parser::NetflowParser;
937 ///
938 /// let parser = NetflowParser::builder()
939 /// .with_cache_size(2000)
940 /// .build()
941 /// .expect("Failed to build parser");
942 /// ```
943 pub fn build(self) -> Result<NetflowParser, String> {
944 let v9_parser =
945 V9Parser::try_new(self.v9_config).map_err(|e| format!("V9 parser error: {}", e))?;
946 let ipfix_parser = IPFixParser::try_new(self.ipfix_config)
947 .map_err(|e| format!("IPFIX parser error: {}", e))?;
948
949 Ok(NetflowParser {
950 v9_parser,
951 ipfix_parser,
952 allowed_versions: self.allowed_versions,
953 max_error_sample_size: self.max_error_sample_size,
954 })
955 }
956}
957
958#[derive(Debug, Clone)]
959pub enum ParsedNetflow<'a> {
960 Success {
961 packet: NetflowPacket,
962 remaining: &'a [u8],
963 },
964 Error {
965 error: NetflowParseError,
966 },
967 UnallowedVersion,
968}
969
970#[derive(Debug, Clone, Serialize)]
971pub struct NetflowPacketError {
972 pub error: NetflowParseError,
973 pub remaining: Vec<u8>,
974}
975
976#[derive(Debug, Clone, Serialize)]
977pub enum NetflowParseError {
978 Incomplete(String),
979 Partial(PartialParse),
980 UnknownVersion(Vec<u8>),
981}
982
983#[derive(Debug, Clone, Serialize)]
984pub struct PartialParse {
985 pub version: u16,
986 pub remaining: Vec<u8>,
987 pub error: String,
988}
989
990/// Iterator that yields NetflowPacket items from a byte buffer without allocating a Vec.
991/// Maintains parser state for template caching (V9/IPFIX).
992pub struct NetflowPacketIterator<'a> {
993 parser: &'a mut NetflowParser,
994 remaining: &'a [u8],
995 errored: bool,
996}
997
998impl<'a> NetflowPacketIterator<'a> {
999 /// Returns the unconsumed bytes remaining in the buffer.
1000 ///
1001 /// This is useful for:
1002 /// - Debugging: See how much data was consumed
1003 /// - Mixed protocols: Process non-netflow data after netflow packets
1004 /// - Resumption: Know where parsing stopped
1005 ///
1006 /// # Examples
1007 ///
1008 /// ```rust
1009 /// use netflow_parser::NetflowParser;
1010 ///
1011 /// let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
1012 /// let mut parser = NetflowParser::default();
1013 /// let mut iter = parser.iter_packets(&v5_packet);
1014 ///
1015 /// while let Some(_packet) = iter.next() {
1016 /// // Process packet
1017 /// }
1018 ///
1019 /// // Check how many bytes remain unconsumed
1020 /// assert_eq!(iter.remaining().len(), 0);
1021 /// ```
1022 pub fn remaining(&self) -> &'a [u8] {
1023 self.remaining
1024 }
1025
1026 /// Returns true if all bytes have been consumed or an error occurred.
1027 ///
1028 /// This is useful for validation and ensuring complete buffer processing.
1029 ///
1030 /// # Examples
1031 ///
1032 /// ```rust
1033 /// use netflow_parser::NetflowParser;
1034 ///
1035 /// let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
1036 /// let mut parser = NetflowParser::default();
1037 /// let mut iter = parser.iter_packets(&v5_packet);
1038 ///
1039 /// // Consume all packets
1040 /// for _packet in &mut iter {
1041 /// // Process packet
1042 /// }
1043 ///
1044 /// assert!(iter.is_complete());
1045 /// ```
1046 pub fn is_complete(&self) -> bool {
1047 self.remaining.is_empty() || self.errored
1048 }
1049}
1050
1051impl<'a> Iterator for NetflowPacketIterator<'a> {
1052 type Item = NetflowPacket;
1053
1054 fn next(&mut self) -> Option<Self::Item> {
1055 // Stop if we've errored or no bytes remain
1056 if self.errored || self.remaining.is_empty() {
1057 return None;
1058 }
1059
1060 match self.parser.parse_packet_by_version(self.remaining) {
1061 ParsedNetflow::Success {
1062 packet,
1063 remaining: new_remaining,
1064 } => {
1065 self.remaining = new_remaining;
1066 Some(packet)
1067 }
1068 ParsedNetflow::UnallowedVersion => {
1069 self.errored = true;
1070 None
1071 }
1072 ParsedNetflow::Error { error } => {
1073 self.errored = true;
1074 // Only include first N bytes of remaining data in error to prevent memory exhaustion
1075 let remaining_sample =
1076 if self.remaining.len() > self.parser.max_error_sample_size {
1077 self.remaining[..self.parser.max_error_sample_size].to_vec()
1078 } else {
1079 self.remaining.to_vec()
1080 };
1081 Some(NetflowPacket::Error(NetflowPacketError {
1082 error,
1083 remaining: remaining_sample,
1084 }))
1085 }
1086 }
1087 }
1088}
1089
1090impl Default for NetflowParser {
1091 fn default() -> Self {
1092 Self {
1093 v9_parser: V9Parser::default(),
1094 ipfix_parser: IPFixParser::default(),
1095 allowed_versions: [5, 7, 9, 10].iter().cloned().collect(),
1096 max_error_sample_size: 256,
1097 }
1098 }
1099}
1100
1101impl NetflowParser {
1102 /// Creates a new builder for configuring a [`NetflowParser`].
1103 ///
1104 /// # Examples
1105 ///
1106 /// ```rust
1107 /// use netflow_parser::NetflowParser;
1108 /// use netflow_parser::variable_versions::ttl::TtlConfig;
1109 /// use std::time::Duration;
1110 ///
1111 /// let parser = NetflowParser::builder()
1112 /// .with_cache_size(2000)
1113 /// .with_ttl(TtlConfig::new(Duration::from_secs(7200)))
1114 /// .build()
1115 /// .expect("Failed to build parser");
1116 /// ```
1117 pub fn builder() -> NetflowParserBuilder {
1118 NetflowParserBuilder::default()
1119 }
1120
1121 /// Gets statistics about the V9 template cache.
1122 ///
1123 /// # Examples
1124 ///
1125 /// ```rust
1126 /// use netflow_parser::NetflowParser;
1127 ///
1128 /// let parser = NetflowParser::default();
1129 /// let stats = parser.v9_cache_stats();
1130 /// println!("V9 cache: {}/{} templates", stats.current_size, stats.max_size);
1131 /// ```
1132 pub fn v9_cache_stats(&self) -> CacheStats {
1133 CacheStats {
1134 current_size: self.v9_parser.templates.len()
1135 + self.v9_parser.options_templates.len(),
1136 max_size: self.v9_parser.max_template_cache_size,
1137 ttl_config: self.v9_parser.ttl_config.clone(),
1138 }
1139 }
1140
1141 /// Gets statistics about the IPFIX template cache.
1142 ///
1143 /// # Examples
1144 ///
1145 /// ```rust
1146 /// use netflow_parser::NetflowParser;
1147 ///
1148 /// let parser = NetflowParser::default();
1149 /// let stats = parser.ipfix_cache_stats();
1150 /// println!("IPFIX cache: {}/{} templates", stats.current_size, stats.max_size);
1151 /// ```
1152 pub fn ipfix_cache_stats(&self) -> CacheStats {
1153 CacheStats {
1154 current_size: self.ipfix_parser.templates.len()
1155 + self.ipfix_parser.v9_templates.len()
1156 + self.ipfix_parser.ipfix_options_templates.len()
1157 + self.ipfix_parser.v9_options_templates.len(),
1158 max_size: self.ipfix_parser.max_template_cache_size,
1159 ttl_config: self.ipfix_parser.ttl_config.clone(),
1160 }
1161 }
1162
1163 /// Lists all cached V9 template IDs.
1164 ///
1165 /// Note: This returns template IDs from both regular and options templates.
1166 ///
1167 /// # Examples
1168 ///
1169 /// ```rust
1170 /// use netflow_parser::NetflowParser;
1171 ///
1172 /// let parser = NetflowParser::default();
1173 /// let template_ids = parser.v9_template_ids();
1174 /// println!("Cached V9 templates: {:?}", template_ids);
1175 /// ```
1176 pub fn v9_template_ids(&self) -> Vec<u16> {
1177 let mut ids: Vec<u16> = self.v9_parser.templates.iter().map(|(id, _)| *id).collect();
1178 ids.extend(self.v9_parser.options_templates.iter().map(|(id, _)| *id));
1179 ids.sort_unstable();
1180 ids
1181 }
1182
1183 /// Lists all cached IPFIX template IDs.
1184 ///
1185 /// Note: This returns template IDs from both IPFIX and V9-format templates (IPFIX can contain both).
1186 ///
1187 /// # Examples
1188 ///
1189 /// ```rust
1190 /// use netflow_parser::NetflowParser;
1191 ///
1192 /// let parser = NetflowParser::default();
1193 /// let template_ids = parser.ipfix_template_ids();
1194 /// println!("Cached IPFIX templates: {:?}", template_ids);
1195 /// ```
1196 pub fn ipfix_template_ids(&self) -> Vec<u16> {
1197 let mut ids: Vec<u16> = self
1198 .ipfix_parser
1199 .templates
1200 .iter()
1201 .map(|(id, _)| *id)
1202 .collect();
1203 ids.extend(self.ipfix_parser.v9_templates.iter().map(|(id, _)| *id));
1204 ids.extend(
1205 self.ipfix_parser
1206 .ipfix_options_templates
1207 .iter()
1208 .map(|(id, _)| *id),
1209 );
1210 ids.extend(
1211 self.ipfix_parser
1212 .v9_options_templates
1213 .iter()
1214 .map(|(id, _)| *id),
1215 );
1216 ids.sort_unstable();
1217 ids
1218 }
1219
1220 /// Checks if a V9 template with the given ID is cached.
1221 ///
1222 /// Note: This uses `peek()` which does not affect LRU ordering.
1223 ///
1224 /// # Arguments
1225 ///
1226 /// * `template_id` - The template ID to check
1227 ///
1228 /// # Examples
1229 ///
1230 /// ```rust
1231 /// use netflow_parser::NetflowParser;
1232 ///
1233 /// let parser = NetflowParser::default();
1234 /// if parser.has_v9_template(256) {
1235 /// println!("Template 256 is cached");
1236 /// }
1237 /// ```
1238 pub fn has_v9_template(&self, template_id: u16) -> bool {
1239 self.v9_parser.templates.peek(&template_id).is_some()
1240 || self
1241 .v9_parser
1242 .options_templates
1243 .peek(&template_id)
1244 .is_some()
1245 }
1246
1247 /// Checks if an IPFIX template with the given ID is cached.
1248 ///
1249 /// Note: This uses `peek()` which does not affect LRU ordering.
1250 ///
1251 /// # Arguments
1252 ///
1253 /// * `template_id` - The template ID to check
1254 ///
1255 /// # Examples
1256 ///
1257 /// ```rust
1258 /// use netflow_parser::NetflowParser;
1259 ///
1260 /// let parser = NetflowParser::default();
1261 /// if parser.has_ipfix_template(256) {
1262 /// println!("Template 256 is cached");
1263 /// }
1264 /// ```
1265 pub fn has_ipfix_template(&self, template_id: u16) -> bool {
1266 self.ipfix_parser.templates.peek(&template_id).is_some()
1267 || self.ipfix_parser.v9_templates.peek(&template_id).is_some()
1268 || self
1269 .ipfix_parser
1270 .ipfix_options_templates
1271 .peek(&template_id)
1272 .is_some()
1273 || self
1274 .ipfix_parser
1275 .v9_options_templates
1276 .peek(&template_id)
1277 .is_some()
1278 }
1279
1280 /// Clears all cached V9 templates.
1281 ///
1282 /// This is useful for testing or when you need to force template re-learning.
1283 ///
1284 /// # Examples
1285 ///
1286 /// ```rust
1287 /// use netflow_parser::NetflowParser;
1288 ///
1289 /// let mut parser = NetflowParser::default();
1290 /// parser.clear_v9_templates();
1291 /// ```
1292 pub fn clear_v9_templates(&mut self) {
1293 self.v9_parser.templates.clear();
1294 self.v9_parser.options_templates.clear();
1295 }
1296
1297 /// Clears all cached IPFIX templates.
1298 ///
1299 /// This is useful for testing or when you need to force template re-learning.
1300 ///
1301 /// # Examples
1302 ///
1303 /// ```rust
1304 /// use netflow_parser::NetflowParser;
1305 ///
1306 /// let mut parser = NetflowParser::default();
1307 /// parser.clear_ipfix_templates();
1308 /// ```
1309 pub fn clear_ipfix_templates(&mut self) {
1310 self.ipfix_parser.templates.clear();
1311 self.ipfix_parser.v9_templates.clear();
1312 self.ipfix_parser.ipfix_options_templates.clear();
1313 self.ipfix_parser.v9_options_templates.clear();
1314 }
1315
1316 /// Takes a Netflow packet slice and returns a vector of Parsed Netflows.
1317 /// If we reach some parse error we return what items be have.
1318 ///
1319 /// # Examples
1320 ///
1321 /// ```rust
1322 /// use serde_json::json;
1323 /// use netflow_parser::NetflowParser;
1324 ///
1325 /// let v5_packet = [0, 5, 2, 0, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
1326 /// println!("{}", json!(NetflowParser::default().parse_bytes(&v5_packet)).to_string());
1327 /// ```
1328 ///
1329 /// ## Output:
1330 ///
1331 /// ```json
1332 /// [{"V5":{"header":{"count":1,"engine_id":7,"engine_type":6,"flow_sequence":33752069,"sampling_interval":2057,"sys_up_time":{"nanos":672000000,"secs":50332},"unix_nsecs":134807553,"unix_secs":83887623,"version":5},"sets":[{"d_octets":66051,"d_pkts":101124105,"dst_addr":"4.5.6.7","dst_as":515,"dst_mask":5,"dst_port":1029,"first":{"nanos":87000000,"secs":67438},"input":515,"last":{"nanos":553000000,"secs":134807},"next_hop":"8.9.0.1","output":1029,"pad1":6,"pad2":1543,"protocol_number":8,"protocol_type":"Egp","src_addr":"0.1.2.3","src_as":1,"src_mask":4,"src_port":515,"tcp_flags":7,"tos":9}]}}]
1333 /// ```
1334 ///
1335 #[inline]
1336 pub fn parse_bytes(&mut self, packet: &[u8]) -> Vec<NetflowPacket> {
1337 if packet.is_empty() {
1338 return vec![];
1339 }
1340
1341 let mut packets = Vec::new();
1342 let mut remaining = packet;
1343
1344 while !remaining.is_empty() {
1345 match self.parse_packet_by_version(remaining) {
1346 ParsedNetflow::Success {
1347 packet,
1348 remaining: new_remaining,
1349 } => {
1350 packets.push(packet);
1351 remaining = new_remaining;
1352 }
1353 ParsedNetflow::UnallowedVersion => {
1354 break;
1355 }
1356 ParsedNetflow::Error { error } => {
1357 // Only include first N bytes of remaining data in error to prevent memory exhaustion
1358 let remaining_sample = if remaining.len() > self.max_error_sample_size {
1359 remaining[..self.max_error_sample_size].to_vec()
1360 } else {
1361 remaining.to_vec()
1362 };
1363 packets.push(NetflowPacket::Error(NetflowPacketError {
1364 error,
1365 remaining: remaining_sample,
1366 }));
1367 break;
1368 }
1369 }
1370 }
1371
1372 packets
1373 }
1374
1375 /// Returns an iterator that yields NetflowPacket items without allocating a Vec.
1376 /// This is useful for processing large batches of packets without collecting all results in memory.
1377 ///
1378 /// # Examples
1379 ///
1380 /// ```rust
1381 /// use netflow_parser::{NetflowParser, NetflowPacket};
1382 ///
1383 /// let v5_packet = [0, 5, 0, 1, 3, 0, 4, 0, 5, 0, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7,];
1384 /// let mut parser = NetflowParser::default();
1385 ///
1386 /// for packet in parser.iter_packets(&v5_packet) {
1387 /// match packet {
1388 /// NetflowPacket::V5(v5) => println!("V5 packet: {:?}", v5.header.version),
1389 /// NetflowPacket::Error(e) => println!("Error: {:?}", e),
1390 /// _ => (),
1391 /// }
1392 /// }
1393 /// ```
1394 #[inline]
1395 pub fn iter_packets<'a>(&'a mut self, packet: &'a [u8]) -> NetflowPacketIterator<'a> {
1396 NetflowPacketIterator {
1397 parser: self,
1398 remaining: packet,
1399 errored: false,
1400 }
1401 }
1402
1403 #[inline]
1404 fn parse_packet_by_version<'a>(&mut self, packet: &'a [u8]) -> ParsedNetflow<'a> {
1405 match GenericNetflowHeader::parse(packet) {
1406 Ok((packet, header)) if self.allowed_versions.contains(&header.version) => {
1407 match header.version {
1408 5 => V5Parser::parse(packet),
1409 7 => V7Parser::parse(packet),
1410 9 => self.v9_parser.parse(packet),
1411 10 => self.ipfix_parser.parse(packet),
1412 _ => ParsedNetflow::Error {
1413 error: NetflowParseError::UnknownVersion(packet.to_vec()),
1414 },
1415 }
1416 }
1417 Ok((_, _)) => ParsedNetflow::UnallowedVersion,
1418 Err(e) => ParsedNetflow::Error {
1419 error: NetflowParseError::Incomplete(e.to_string()),
1420 },
1421 }
1422 }
1423
1424 /// Takes a Netflow packet slice and returns a vector of Parsed NetflowCommonFlowSet
1425 #[cfg(feature = "netflow_common")]
1426 #[inline]
1427 pub fn parse_bytes_as_netflow_common_flowsets(
1428 &mut self,
1429 packet: &[u8],
1430 ) -> Vec<NetflowCommonFlowSet> {
1431 let netflow_packets = self.parse_bytes(packet);
1432 netflow_packets
1433 .iter()
1434 .flat_map(|n| n.as_netflow_common().unwrap_or_default().flowsets)
1435 .collect()
1436 }
1437}