1use crate::db::Label;
2use crate::error::PassiveTcpError;
3use crate::observable_signals::{ObservableHttpRequest, ObservableHttpResponse};
4use crate::{http, http_languages};
5use httparse::{Request, Response, EMPTY_HEADER};
6use pnet::packet::ip::IpNextHeaderProtocols;
7use pnet::packet::ipv4::Ipv4Packet;
8use pnet::packet::ipv6::Ipv6Packet;
9use pnet::packet::tcp::TcpPacket;
10use pnet::packet::Packet;
11use std::net::IpAddr;
12use std::time::Duration;
13use tracing::debug;
14use ttl_cache::TtlCache;
15
16const HTTP_MAX_HDRS: usize = 32;
18
19pub type FlowKey = (IpAddr, IpAddr, u16, u16);
21
22pub struct ObservableHttpPackage {
23 pub http_request: Option<ObservableHttpRequest>,
24 pub http_response: Option<ObservableHttpResponse>,
25}
26
27#[derive(Clone)]
28struct TcpData {
29 sequence: u32,
30 data: Vec<u8>,
31}
32
33pub struct TcpFlow {
34 client_ip: IpAddr,
35 server_ip: IpAddr,
36 client_port: u16,
37 server_port: u16,
38 client_data: Vec<TcpData>,
39 server_data: Vec<TcpData>,
40}
41impl TcpFlow {
42 fn init(
43 src_ip: IpAddr,
44 src_port: u16,
45 dst_ip: IpAddr,
46 dst_port: u16,
47 tcp_data: TcpData,
48 ) -> TcpFlow {
49 TcpFlow {
50 client_ip: src_ip,
51 server_ip: dst_ip,
52 client_port: src_port,
53 server_port: dst_port,
54 client_data: vec![tcp_data.clone()],
55 server_data: Vec::new(),
56 }
57 }
58 fn get_full_data(&self, is_client: bool) -> Vec<u8> {
63 let data: &Vec<TcpData> = if is_client {
64 &self.client_data
65 } else {
66 &self.server_data
67 };
68
69 let mut sorted_data = data.clone();
70
71 sorted_data.sort_by_key(|tcp_data| tcp_data.sequence);
72
73 let mut full_data = Vec::new();
74 for tcp_data in sorted_data {
75 full_data.extend_from_slice(&tcp_data.data);
76 }
77 full_data
78 }
79}
80
81pub fn process_http_ipv4(
82 packet: &Ipv4Packet,
83 cache: &mut TtlCache<FlowKey, TcpFlow>,
84) -> Result<ObservableHttpPackage, PassiveTcpError> {
85 if packet.get_next_level_protocol() != IpNextHeaderProtocols::Tcp {
86 return Err(PassiveTcpError::UnsupportedProtocol("IPv4".to_string()));
87 }
88 if let Some(tcp) = TcpPacket::new(packet.payload()) {
89 process_tcp_packet(
90 cache,
91 tcp,
92 IpAddr::V4(packet.get_source()),
93 IpAddr::V4(packet.get_destination()),
94 )
95 } else {
96 Ok(ObservableHttpPackage {
97 http_request: None,
98 http_response: None,
99 })
100 }
101}
102
103pub fn process_http_ipv6(
104 packet: &Ipv6Packet,
105 cache: &mut TtlCache<FlowKey, TcpFlow>,
106) -> Result<ObservableHttpPackage, PassiveTcpError> {
107 if packet.get_next_header() != IpNextHeaderProtocols::Tcp {
108 return Err(PassiveTcpError::UnsupportedProtocol("IPv6".to_string()));
109 }
110 if let Some(tcp) = TcpPacket::new(packet.payload()) {
111 process_tcp_packet(
112 cache,
113 tcp,
114 IpAddr::V6(packet.get_source()),
115 IpAddr::V6(packet.get_destination()),
116 )
117 } else {
118 Ok(ObservableHttpPackage {
119 http_request: None,
120 http_response: None,
121 })
122 }
123}
124
125fn process_tcp_packet(
126 cache: &mut TtlCache<FlowKey, TcpFlow>,
127 tcp: TcpPacket,
128 src_ip: IpAddr,
129 dst_ip: IpAddr,
130) -> Result<ObservableHttpPackage, PassiveTcpError> {
131 let src_port: u16 = tcp.get_source();
132 let dst_port: u16 = tcp.get_destination();
133 let mut observable_http_package = ObservableHttpPackage {
134 http_request: None,
135 http_response: None,
136 };
137
138 let flow_key: FlowKey = (src_ip, dst_ip, src_port, dst_port);
139 let (tcp_flow, is_client) = {
140 if let Some(flow) = cache.get_mut(&flow_key) {
141 (Some(flow), true)
142 } else {
143 let reversed_key: FlowKey = (dst_ip, src_ip, dst_port, src_port);
144 if let Some(flow) = cache.get_mut(&reversed_key) {
145 (Some(flow), false)
146 } else {
147 (None, false)
148 }
149 }
150 };
151
152 if let Some(flow) = tcp_flow {
153 if !tcp.payload().is_empty() {
154 let tcp_data = TcpData {
155 sequence: tcp.get_sequence(),
156 data: Vec::from(tcp.payload()),
157 };
158
159 if is_client && src_ip == flow.client_ip && src_port == flow.client_port {
160 flow.client_data.push(tcp_data);
161 if let Ok(http_request_parsed) = parse_http_request(&flow.get_full_data(is_client))
162 {
163 observable_http_package.http_request = http_request_parsed;
164 }
165 } else if src_ip == flow.server_ip && src_port == flow.server_port {
166 flow.server_data.push(tcp_data);
167 if let Ok(http_response_parsed) =
168 parse_http_response(&flow.get_full_data(is_client))
169 {
170 observable_http_package.http_response = http_response_parsed;
171
172 if tcp.get_flags()
173 & (pnet::packet::tcp::TcpFlags::FIN | pnet::packet::tcp::TcpFlags::RST)
174 != 0
175 {
176 debug!("Connection closed or reset");
177 cache.remove(&flow_key);
178 }
179 }
180 }
181 }
182 } else if tcp.get_flags() & pnet::packet::tcp::TcpFlags::SYN != 0 {
183 let tcp_data: TcpData = TcpData {
184 sequence: tcp.get_sequence(),
185 data: Vec::from(tcp.payload()),
186 };
187 let flow: TcpFlow = TcpFlow::init(src_ip, src_port, dst_ip, dst_port, tcp_data);
188 cache.insert(flow_key, flow, Duration::new(60, 0));
189 }
190
191 Ok(observable_http_package)
192}
193
194fn parse_http_request(data: &[u8]) -> Result<Option<ObservableHttpRequest>, PassiveTcpError> {
195 let mut headers = [EMPTY_HEADER; HTTP_MAX_HDRS];
196 let mut req = Request::new(&mut headers);
197
198 match req.parse(data) {
199 Ok(httparse::Status::Complete(_)) => {
200 let headers: &[httparse::Header] = req.headers;
201
202 let headers_in_order: Vec<http::Header> = build_headers_in_order(headers, true);
203 let headers_absent: Vec<http::Header> = build_headers_absent_in_order(headers, true);
204 let user_agent: Option<String> = extract_header_by_name(headers, "User-Agent");
205 let lang: Option<String> =
206 extract_header_by_name(headers, "Accept-Language").and_then(|accept_language| {
207 http_languages::get_highest_quality_language(accept_language)
208 });
209 let http_version: http::Version = extract_http_version(req.version);
210
211 Ok(Some(ObservableHttpRequest {
212 lang,
213 user_agent: user_agent.clone(),
214 version: http_version,
215 horder: headers_in_order,
216 habsent: headers_absent,
217 expsw: extract_traffic_classification(user_agent),
218 }))
219 }
220 Ok(httparse::Status::Partial) => {
221 debug!("Incomplete HTTP request data. Data: {:?}", data);
222 Ok(None)
223 }
224 Err(e) => {
225 debug!(
226 "Failed to parse HTTP request with Data: {:?}. Error: {}",
227 data, e
228 );
229 Err(PassiveTcpError::Parse(format!(
230 "Failed to parse HTTP request: {}",
231 e
232 )))
233 }
234 }
235}
236
237fn parse_http_response(data: &[u8]) -> Result<Option<ObservableHttpResponse>, PassiveTcpError> {
238 let mut headers = [EMPTY_HEADER; HTTP_MAX_HDRS];
239 let mut res = Response::new(&mut headers);
240
241 match res.parse(data) {
242 Ok(httparse::Status::Complete(_)) => {
243 let headers: &[httparse::Header] = res.headers;
244
245 let headers_in_order: Vec<http::Header> = build_headers_in_order(headers, false);
246 let headers_absent: Vec<http::Header> = build_headers_absent_in_order(headers, false);
247 let server: Option<String> = extract_header_by_name(headers, "Server");
248 let http_version: http::Version = extract_http_version(res.version);
249
250 Ok(Some(ObservableHttpResponse {
251 version: http_version,
252 horder: headers_in_order,
253 habsent: headers_absent,
254 expsw: extract_traffic_classification(server),
255 }))
256 }
257 Ok(httparse::Status::Partial) => {
258 debug!("Incomplete HTTP response data. Data: {:?}", data);
259 Ok(None)
260 }
261 Err(e) => {
262 debug!(
263 "Failed to parse HTTP response with Data: {:?}. Error: {}",
264 data, e
265 );
266 Err(PassiveTcpError::Parse(format!(
267 "Failed to parse HTTP response: {}",
268 e
269 )))
270 }
271 }
272}
273
274fn build_headers_in_order(headers: &[httparse::Header], is_request: bool) -> Vec<http::Header> {
275 let mut headers_in_order: Vec<http::Header> = Vec::new();
276 let optional_list = if is_request {
277 http::request_optional_headers()
278 } else {
279 http::response_optional_headers()
280 };
281 let skip_value_list = if is_request {
282 http::request_skip_value_headers()
283 } else {
284 http::response_skip_value_headers()
285 };
286 let common_list = if is_request {
287 http::request_common_headers()
288 } else {
289 http::response_common_headers()
290 };
291
292 #[allow(clippy::if_same_then_else)]
293 for header in headers {
294 let value: Option<&str> = std::str::from_utf8(header.value).ok();
295
296 if optional_list.contains(&header.name) {
297 headers_in_order.push(http::Header::new(header.name).optional());
298 } else if skip_value_list.contains(&header.name) {
299 headers_in_order.push(http::Header::new(header.name));
300 } else if common_list.contains(&header.name) {
301 headers_in_order.push(http::Header::new(header.name).with_optional_value(value));
302 } else {
303 headers_in_order.push(http::Header::new(header.name).with_optional_value(value));
304 }
305 }
306
307 headers_in_order
308}
309
310fn build_headers_absent_in_order(
311 headers: &[httparse::Header],
312 is_request: bool,
313) -> Vec<http::Header> {
314 let mut headers_absent: Vec<http::Header> = Vec::new();
315 let common_list: Vec<&str> = if is_request {
316 http::request_common_headers()
317 } else {
318 http::response_common_headers()
319 };
320 let current_headers: Vec<&str> = headers.iter().map(|h| h.name).collect();
321
322 for header in &common_list {
323 if !current_headers.contains(header) {
324 headers_absent.push(http::Header::new(header));
325 }
326 }
327 headers_absent
328}
329
330fn extract_header_by_name(headers: &[httparse::Header], name: &str) -> Option<String> {
331 headers
332 .iter()
333 .find(|header| header.name.eq_ignore_ascii_case(name))
334 .map(|header| String::from_utf8_lossy(header.value).to_string())
335}
336
337fn extract_traffic_classification(value: Option<String>) -> String {
338 value.unwrap_or_else(|| "???".to_string())
339}
340
341fn extract_http_version(version: Option<u8>) -> http::Version {
342 match version {
343 Some(0) => http::Version::V10,
344 Some(1) => http::Version::V11,
345 _ => http::Version::Any,
346 }
347}
348
349pub fn get_diagnostic(
350 user_agent: Option<String>,
351 ua_matcher: Option<(&String, &Option<String>)>,
352 signature_os_matcher: Option<&Label>,
353) -> http::HttpDiagnosis {
354 match user_agent {
355 None => http::HttpDiagnosis::Anonymous,
356 Some(_ua) => match (ua_matcher, signature_os_matcher) {
357 (Some((ua_name_db, _ua_flavor_db)), Some(signature_label_db)) => {
358 if ua_name_db.eq_ignore_ascii_case(&signature_label_db.name) {
359 http::HttpDiagnosis::Generic
360 } else {
361 http::HttpDiagnosis::Dishonest
362 }
363 }
364 _ => http::HttpDiagnosis::None,
365 },
366 }
367}
368
369#[cfg(test)]
370mod tests {
371 use super::*;
372 use crate::db;
373
374 #[test]
375 fn test_parse_http_request() {
376 let valid_request = b"GET / HTTP/1.1\r\n\
377 Host: example.com\r\n\
378 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\r\n\
379 Accept-Language: en-US,en;q=0.9,es;q=0.8\r\n\
380 Cache-Control: max-age=0\r\n\
381 Connection: keep-alive\r\n\
382 If-Modified-Since: Thu, 17 Oct 2019 07:18:26 GMT\r\n\
383 If-None-Match: \"3147526947\"\r\n\
384 Upgrade-Insecure-Requests: 1\r\n\
385 User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36\r\n\
386 \r\n";
387 match parse_http_request(valid_request) {
388 Ok(Some(request)) => {
389 assert_eq!(request.lang, Some("English".to_string()));
390 assert_eq!(request.user_agent, Some("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36".to_string()));
391 assert_eq!(request.version, http::Version::V11);
392
393 let expected_horder = vec![
394 http::Header::new("Host"),
395 http::Header::new("Accept").with_value("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"),
396 http::Header::new("Accept-Language").with_value("en-US,en;q=0.9,es;q=0.8"),
397 http::Header::new("Cache-Control").optional(),
398 http::Header::new("Connection").with_value("keep-alive"),
399 http::Header::new("If-Modified-Since").optional(),
400 http::Header::new("If-None-Match").optional(),
401 http::Header::new("Upgrade-Insecure-Requests").with_value("1"),
402 http::Header::new("User-Agent"),
403 ];
404 assert_eq!(request.horder, expected_horder);
405
406 let expected_habsent = vec![
407 http::Header::new("Accept-Encoding"),
408 http::Header::new("Accept-Charset"),
409 http::Header::new("Keep-Alive"),
410 ];
411 assert_eq!(request.habsent, expected_habsent);
412
413 assert_eq!(request.expsw, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36");
414 }
415 Ok(None) => panic!("Incomplete HTTP request"),
416 Err(e) => panic!("Failed to parse HTTP request: {}", e),
417 }
418 }
419
420 #[test]
421 fn test_parse_http_response() {
422 let valid_response = b"HTTP/1.1 200 OK\r\n\
423 Server: Apache\r\n\
424 Content-Type: text/html; charset=UTF-8\r\n\
425 Content-Length: 112\r\n\
426 Connection: keep-alive\r\n\
427 \r\n\
428 <html><body><h1>It works!</h1></body></html>";
429
430 match parse_http_response(valid_response) {
431 Ok(Some(response)) => {
432 assert_eq!(response.expsw, "Apache");
433 assert_eq!(response.version, http::Version::V11);
434
435 let expected_horder = vec![
436 http::Header::new("Server"),
437 http::Header::new("Content-Type"),
438 http::Header::new("Content-Length").optional(),
439 http::Header::new("Connection").with_value("keep-alive"),
440 ];
441 assert_eq!(response.horder, expected_horder);
442
443 let expected_absent = vec![
444 http::Header::new("Keep-Alive"),
445 http::Header::new("Accept-Ranges"),
446 http::Header::new("Date"),
447 ];
448 assert_eq!(response.habsent, expected_absent);
449 }
450 Ok(None) => panic!("Incomplete HTTP response"),
451 Err(e) => panic!("Failed to parse HTTP response: {}", e),
452 }
453 }
454
455 #[test]
456 fn test_get_diagnostic_for_empty_sw() {
457 let diagnosis: http::HttpDiagnosis = get_diagnostic(None, None, None);
458 assert_eq!(diagnosis, http::HttpDiagnosis::Anonymous);
459 }
460
461 #[test]
462 fn test_get_diagnostic_with_existing_signature_matcher() {
463 let user_agent: Option<String> = Some("Mozilla/5.0".to_string());
464 let os = "Linux".to_string();
465 let browser = Some("Firefox".to_string());
466 let ua_matcher: Option<(&String, &Option<String>)> = Some((&os, &browser));
467 let label = Label {
468 ty: db::Type::Specified,
469 class: None,
470 name: "Linux".to_string(),
471 flavor: None,
472 };
473 let signature_os_matcher: Option<&Label> = Some(&label);
474
475 let diagnosis = get_diagnostic(user_agent, ua_matcher, signature_os_matcher);
476 assert_eq!(diagnosis, http::HttpDiagnosis::Generic);
477 }
478
479 #[test]
480 fn test_get_diagnostic_with_dishonest_user_agent() {
481 let user_agent = Some("Mozilla/5.0".to_string());
482 let os = "Windows".to_string();
483 let browser = Some("Firefox".to_string());
484 let ua_matcher: Option<(&String, &Option<String>)> = Some((&os, &browser));
485 let label = Label {
486 ty: db::Type::Specified,
487 class: None,
488 name: "Linux".to_string(),
489 flavor: None,
490 };
491 let signature_os_matcher: Option<&Label> = Some(&label);
492
493 let diagnosis = get_diagnostic(user_agent, ua_matcher, signature_os_matcher);
494 assert_eq!(diagnosis, http::HttpDiagnosis::Dishonest);
495 }
496
497 #[test]
498 fn test_get_diagnostic_without_user_agent_and_signature_matcher() {
499 let user_agent = Some("Mozilla/5.0".to_string());
500
501 let diagnosis = get_diagnostic(user_agent, None, None);
502 assert_eq!(diagnosis, http::HttpDiagnosis::None);
503 }
504}