cloudfront_logs/owned/
parquet.rs

1use crate::{
2    borrowed::raw::{UnvalidatedLogline as UnvalidatedRaw, ValidatedLogline as ValidatedRaw},
3    shared::*,
4    types::*,
5    CHRONO_DATE_FMT, CHRONO_TIME_FMT,
6};
7
8pub use crate::types::{Datelike, Timelike};
9
10/// The validated log line for [`parquet`] usage
11///
12/// Most fields are parsed into more meaningful types.
13/// Unfortunately, [`parquet_derive`] does not support all the types;
14/// thus we lower some fields down to:
15/// * &str / Option<&str> (instead of enums, NaiveTime, IpAddr)
16/// * f64 (instead of Duration)
17///
18/// On construction it checks if the line can be parsed.
19/// This is useful if you cannot skip the comment lines or have reason to not trust the input for format correctness.
20/// The latter should be only an issue if you do not use this crate on CloudFront logs directly.
21///
22/// # Panics
23///
24/// Construction can panic if the input is not a valid log line!
25///
26/// # Examples
27///
28/// Use `.try_from()` or `.try_into()` to construct an instance, since action can fail.
29///
30/// ```rust
31/// use cloudfront_logs::{borrowed::parquet::ValidatedLogline, types::*};
32///
33/// let line = "2019-12-04	21:02:31	LAX1	392	192.0.2.100	GET	d111111abcdef8.cloudfront.net	/index.html	200	-	Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36	-	-	Hit	SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==	d111111abcdef8.cloudfront.net	https	23	0.001	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	Hit	HTTP/2.0	-	-	11040	0.001	Hit	text/html	78	-	-";
34///
35/// let item = ValidatedLogline::try_from(line).unwrap();
36/// // alternative:
37/// let item: ValidatedLogline<'_> = line.try_into().unwrap();
38///
39/// assert_eq!(item.date, NaiveDate::from_ymd_opt(2019, 12, 4).unwrap());
40/// assert_eq!(item.sc_bytes, 392u64);
41/// assert_eq!(item.cs_protocol, "https");
42/// ```
43#[must_use]
44#[derive(Debug, Clone, PartialEq, parquet_derive::ParquetRecordWriter)]
45pub struct ValidatedLogline {
46    pub date: NaiveDate,
47    pub time: String, // not supported: NaiveTime
48    pub datetime: NaiveDateTime,
49    pub x_edge_location: String,
50    pub sc_bytes: u64,
51    pub c_ip: String,
52    pub cs_method: String,
53    pub cs_host: String,
54    pub cs_uri_stem: String,
55    pub sc_status: u16,
56    pub cs_referer: Option<String>,
57    pub cs_user_agent: String,
58    pub cs_uri_query: Option<String>,
59    pub cs_cookie: Option<String>,
60    pub x_edge_result_type: String,
61    pub x_edge_request_id: String,
62    pub x_host_header: String,
63    pub cs_protocol: String,
64    pub cs_bytes: u64,
65    pub time_taken: f64,
66    pub x_forwarded_for: Option<String>,
67    pub ssl_protocol: Option<String>,
68    pub ssl_cipher: Option<String>,
69    pub x_edge_response_result_type: String,
70    pub cs_protocol_version: String,
71    pub fle_status: Option<String>,
72    pub fle_encrypted_fields: Option<u64>,
73    pub c_port: u16,
74    pub time_to_first_byte: f64,
75    pub x_edge_detailed_result_type: String,
76    pub sc_content_type: Option<String>,
77    pub sc_content_len: Option<u64>,
78    pub sc_range_start: Option<i64>,
79    pub sc_range_end: Option<i64>,
80}
81
82impl ValidatedLogline {
83    pub fn schema() -> &'static str {
84        crate::consts::parquet_schemata::V1
85    }
86
87    pub fn schema_as_type() -> parquet::schema::types::Type {
88        parquet::schema::parser::parse_message_type(crate::consts::parquet_schemata::V1).unwrap()
89    }
90}
91
92impl TryFrom<&str> for ValidatedLogline {
93    type Error = &'static str;
94
95    fn try_from(line: &str) -> Result<Self, Self::Error> {
96        validate_line(line)?;
97        let mut iter = MemchrTabSplitter::new(line);
98
99        let date = NaiveDate::parse_from_str(iter.next().unwrap(), CHRONO_DATE_FMT)
100            .map_err(|_e| "date invalid")?;
101        let raw_time = iter.next().unwrap();
102        let time =
103            NaiveTime::parse_from_str(raw_time, CHRONO_TIME_FMT).map_err(|_e| "time invalid")?;
104        let datetime = NaiveDateTime::new(date, time);
105
106        let line = Self {
107            date,
108            time: raw_time.to_string(),
109            datetime,
110            x_edge_location: iter.next().unwrap().to_string(),
111            sc_bytes: iter
112                .next()
113                .unwrap()
114                .parse::<u64>()
115                .map_err(|_e| "sc_bytes invalid")?,
116            c_ip: iter.next().unwrap().to_string(),
117            cs_method: iter.next().unwrap().to_string(),
118            cs_host: iter.next().unwrap().to_string(),
119            cs_uri_stem: iter.next().unwrap().to_string(),
120            sc_status: iter
121                .next()
122                .unwrap()
123                .parse::<u16>()
124                .map_err(|_e| "sc_status invalid")?,
125            cs_referer: iter.next().unwrap().to_optional_string(),
126            cs_user_agent: iter.next().unwrap().to_string(),
127            cs_uri_query: iter.next().unwrap().to_optional_string(),
128            cs_cookie: iter.next().unwrap().to_optional_string(),
129            x_edge_result_type: iter.next().unwrap().to_string(),
130            x_edge_request_id: iter.next().unwrap().to_string(),
131            x_host_header: iter.next().unwrap().to_string(),
132            cs_protocol: iter.next().unwrap().to_string(),
133            cs_bytes: iter
134                .next()
135                .unwrap()
136                .parse::<u64>()
137                .map_err(|_e| "cs_bytes invalid")?,
138            time_taken: iter
139                .next()
140                .unwrap()
141                .parse::<f64>()
142                .map_err(|_e| "time_taken invalid")?,
143            x_forwarded_for: iter.next().unwrap().to_optional_string(),
144            ssl_protocol: iter.next().unwrap().to_optional_string(),
145            ssl_cipher: iter.next().unwrap().to_optional_string(),
146            x_edge_response_result_type: iter.next().unwrap().to_string(),
147            cs_protocol_version: iter.next().unwrap().to_string(),
148            fle_status: iter.next().unwrap().to_optional_string(),
149            fle_encrypted_fields: iter
150                .next()
151                .and_then(as_optional_t)
152                .transpose()
153                .map_err(|_e| "fle_encrypted_fields invalid")?,
154            c_port: iter
155                .next()
156                .unwrap()
157                .parse::<u16>()
158                .map_err(|_e| "c_port invalid")?,
159            time_to_first_byte: iter
160                .next()
161                .unwrap()
162                .parse::<f64>()
163                .map_err(|_e| "time_to_first_byte invalid")?,
164            x_edge_detailed_result_type: iter.next().unwrap().to_string(),
165            sc_content_type: iter.next().unwrap().to_optional_string(),
166            sc_content_len: iter
167                .next()
168                .and_then(as_optional_t)
169                .transpose()
170                .map_err(|_e| "sc_content_len invalid")?,
171            sc_range_start: iter
172                .next()
173                .and_then(as_optional_t)
174                .transpose()
175                .map_err(|_e| "sc_range_start invalid")?,
176            sc_range_end: iter
177                .next()
178                .and_then(as_optional_t)
179                .transpose()
180                .map_err(|_e| "sc_range_end invalid")?,
181        };
182        Ok(line)
183    }
184}
185
186impl TryFrom<ValidatedRaw<'_>> for ValidatedLogline {
187    type Error = &'static str;
188
189    fn try_from(raw: ValidatedRaw<'_>) -> Result<Self, Self::Error> {
190        let date =
191            NaiveDate::parse_from_str(raw.date, CHRONO_DATE_FMT).map_err(|_e| "date invalid")?;
192        let time =
193            NaiveTime::parse_from_str(raw.time, CHRONO_TIME_FMT).map_err(|_e| "time invalid")?;
194        let datetime = NaiveDateTime::new(date, time);
195
196        let line = Self {
197            date,
198            time: raw.time.to_string(),
199            datetime,
200            x_edge_location: raw.x_edge_location.to_string(),
201            sc_bytes: raw
202                .sc_bytes
203                .parse::<u64>()
204                .map_err(|_e| "sc_bytes invalid")?,
205            c_ip: raw.c_ip.to_string(),
206            cs_method: raw.cs_method.to_string(),
207            cs_host: raw.cs_host.to_string(),
208            cs_uri_stem: raw.cs_uri_stem.to_string(),
209            sc_status: raw
210                .sc_status
211                .parse::<u16>()
212                .map_err(|_e| "sc_status invalid")?,
213            cs_referer: raw.cs_referer.to_optional_string(),
214            cs_user_agent: raw.cs_user_agent.to_string(),
215            cs_uri_query: raw.cs_uri_query.to_optional_string(),
216            cs_cookie: raw.cs_cookie.to_optional_string(),
217            x_edge_result_type: raw.x_edge_result_type.to_string(),
218            x_edge_request_id: raw.x_edge_request_id.to_string(),
219            x_host_header: raw.x_host_header.to_string(),
220            cs_protocol: raw.cs_protocol.to_string(),
221            cs_bytes: raw
222                .cs_bytes
223                .parse::<u64>()
224                .map_err(|_e| "cs_bytes invalid")?,
225            time_taken: raw
226                .time_taken
227                .parse::<f64>()
228                .map_err(|_e| "time_taken invalid")?,
229            x_forwarded_for: raw.x_forwarded_for.to_optional_string(),
230            ssl_protocol: raw.ssl_protocol.to_optional_string(),
231            ssl_cipher: raw.ssl_cipher.to_optional_string(),
232            x_edge_response_result_type: raw.x_edge_response_result_type.to_string(),
233            cs_protocol_version: raw.cs_protocol_version.to_string(),
234            fle_status: raw.fle_status.to_optional_string(),
235            fle_encrypted_fields: parse_as_option(raw.fle_encrypted_fields)
236                .map_err(|_e| "fle_encrypted_fields invalid")?,
237            c_port: raw.c_port.parse::<u16>().map_err(|_e| "c_port invalid")?,
238            time_to_first_byte: raw
239                .time_to_first_byte
240                .parse::<f64>()
241                .map_err(|_e| "time_to_first_byte invalid")?,
242            x_edge_detailed_result_type: raw.x_edge_detailed_result_type.to_string(),
243            sc_content_type: raw.sc_content_type.to_optional_string(),
244            sc_content_len: parse_as_option(raw.sc_content_len)
245                .map_err(|_e| "sc_content_len invalid")?,
246            sc_range_start: parse_as_option(raw.sc_range_start)
247                .map_err(|_e| "sc_range_start invalid")?,
248            sc_range_end: parse_as_option(raw.sc_range_end).map_err(|_e| "sc_range_end invalid")?,
249        };
250        Ok(line)
251    }
252}
253
254/// The unvalidated log line for [`parquet`] usage
255///
256/// Most fields are parsed into more meaningful types.
257///
258/// Unlike [`ValidatedLogline`], this variant does not check if the line can be parsed.
259/// Use this if you already did a check before creating this struct.
260/// A common scenario is that you 1) trust the input data and 2) skipped the comment lines.
261///
262/// Note: This is the only variant which can use the `From` trait instead of `TryFrom`,
263/// because validation is skipped and the input data does not need to be parsed into other types.
264///
265/// # Panics
266///
267/// Construction can panic if the input is not a valid log line!
268///
269/// # Examples
270///
271/// Use `.try_from()` or `.try_into()` to construct an instance, since action can fail.
272///
273/// ```rust
274/// use cloudfront_logs::{borrowed::parquet::UnvalidatedLogline, types::*};
275///
276/// let line = "2019-12-04	21:02:31	LAX1	392	192.0.2.100	GET	d111111abcdef8.cloudfront.net	/index.html	200	-	Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36	-	-	Hit	SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==	d111111abcdef8.cloudfront.net	https	23	0.001	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	Hit	HTTP/2.0	-	-	11040	0.001	Hit	text/html	78	-	-";
277///
278/// let item = UnvalidatedLogline::try_from(line).unwrap();
279/// // alternative:
280/// let item: UnvalidatedLogline<'_> = line.try_into().unwrap();
281///
282/// assert_eq!(item.date, NaiveDate::from_ymd_opt(2019, 12, 4).unwrap());
283/// assert_eq!(item.sc_bytes, 392u64);
284/// assert_eq!(item.cs_protocol, "https");
285/// ```
286#[must_use]
287#[derive(Debug, Clone, PartialEq, parquet_derive::ParquetRecordWriter)]
288pub struct UnvalidatedLogline {
289    pub date: NaiveDate,
290    pub time: String, // not supported: NaiveTime
291    pub datetime: NaiveDateTime,
292    pub x_edge_location: String,
293    pub sc_bytes: u64,
294    pub c_ip: String,
295    pub cs_method: String,
296    pub cs_host: String,
297    pub cs_uri_stem: String,
298    pub sc_status: u16,
299    pub cs_referer: Option<String>,
300    pub cs_user_agent: String,
301    pub cs_uri_query: Option<String>,
302    pub cs_cookie: Option<String>,
303    pub x_edge_result_type: String,
304    pub x_edge_request_id: String,
305    pub x_host_header: String,
306    pub cs_protocol: String,
307    pub cs_bytes: u64,
308    pub time_taken: f64,
309    pub x_forwarded_for: Option<String>,
310    pub ssl_protocol: Option<String>,
311    pub ssl_cipher: Option<String>,
312    pub x_edge_response_result_type: String,
313    pub cs_protocol_version: String,
314    pub fle_status: Option<String>,
315    pub fle_encrypted_fields: Option<u64>,
316    pub c_port: u16,
317    pub time_to_first_byte: f64,
318    pub x_edge_detailed_result_type: String,
319    pub sc_content_type: Option<String>,
320    pub sc_content_len: Option<u64>,
321    pub sc_range_start: Option<i64>,
322    pub sc_range_end: Option<i64>,
323}
324
325impl UnvalidatedLogline {
326    pub fn schema() -> &'static str {
327        crate::consts::parquet_schemata::V1
328    }
329
330    pub fn schema_as_type() -> parquet::schema::types::Type {
331        parquet::schema::parser::parse_message_type(crate::consts::parquet_schemata::V1).unwrap()
332    }
333}
334
335impl TryFrom<&str> for UnvalidatedLogline {
336    type Error = &'static str;
337
338    fn try_from(line: &str) -> Result<Self, Self::Error> {
339        let mut iter = MemchrTabSplitter::new(line);
340
341        let date = NaiveDate::parse_from_str(iter.next().unwrap(), "%Y-%m-%d")
342            .map_err(|_e| "date invalid")?;
343        let raw_time = iter.next().unwrap();
344        let time = NaiveTime::parse_from_str(raw_time, "%H:%M:%S").map_err(|_e| "time invalid")?;
345        let datetime = NaiveDateTime::new(date, time);
346
347        let line = Self {
348            date,
349            time: raw_time.to_string(),
350            datetime,
351            x_edge_location: iter.next().unwrap().to_string(),
352            sc_bytes: iter
353                .next()
354                .unwrap()
355                .parse::<u64>()
356                .map_err(|_e| "sc_bytes invalid")?,
357            c_ip: iter.next().unwrap().to_string(),
358            cs_method: iter.next().unwrap().to_string(),
359            cs_host: iter.next().unwrap().to_string(),
360            cs_uri_stem: iter.next().unwrap().to_string(),
361            sc_status: iter
362                .next()
363                .unwrap()
364                .parse::<u16>()
365                .map_err(|_e| "sc_status invalid")?,
366            cs_referer: iter.next().unwrap().to_optional_string(),
367            cs_user_agent: iter.next().unwrap().to_string(),
368            cs_uri_query: iter.next().unwrap().to_optional_string(),
369            cs_cookie: iter.next().unwrap().to_optional_string(),
370            x_edge_result_type: iter.next().unwrap().to_string(),
371            x_edge_request_id: iter.next().unwrap().to_string(),
372            x_host_header: iter.next().unwrap().to_string(),
373            cs_protocol: iter.next().unwrap().to_string(),
374            cs_bytes: iter
375                .next()
376                .unwrap()
377                .parse::<u64>()
378                .map_err(|_e| "cs_bytes invalid")?,
379            time_taken: iter
380                .next()
381                .unwrap()
382                .parse::<f64>()
383                .map_err(|_e| "time_taken invalid")?,
384            x_forwarded_for: iter.next().unwrap().to_optional_string(),
385            ssl_protocol: iter.next().unwrap().to_optional_string(),
386            ssl_cipher: iter.next().unwrap().to_optional_string(),
387            x_edge_response_result_type: iter.next().unwrap().to_string(),
388            cs_protocol_version: iter.next().unwrap().to_string(),
389            fle_status: iter.next().unwrap().to_optional_string(),
390            fle_encrypted_fields: iter
391                .next()
392                .and_then(as_optional_t)
393                .transpose()
394                .map_err(|_e| "fle_encrypted_fields invalid")?,
395            c_port: iter
396                .next()
397                .unwrap()
398                .parse::<u16>()
399                .map_err(|_e| "c_port invalid")?,
400            time_to_first_byte: iter
401                .next()
402                .unwrap()
403                .parse::<f64>()
404                .map_err(|_e| "time_to_first_byte invalid")?,
405            x_edge_detailed_result_type: iter.next().unwrap().to_string(),
406            sc_content_type: iter.next().unwrap().to_optional_string(),
407            sc_content_len: iter
408                .next()
409                .and_then(as_optional_t)
410                .transpose()
411                .map_err(|_e| "sc_content_len invalid")?,
412            sc_range_start: iter
413                .next()
414                .and_then(as_optional_t)
415                .transpose()
416                .map_err(|_e| "sc_range_start invalid")?,
417            sc_range_end: iter
418                .next()
419                .and_then(as_optional_t)
420                .transpose()
421                .map_err(|_e| "sc_range_end invalid")?,
422        };
423        Ok(line)
424    }
425}
426
427impl TryFrom<UnvalidatedRaw<'_>> for UnvalidatedLogline {
428    type Error = &'static str;
429
430    fn try_from(raw: UnvalidatedRaw<'_>) -> Result<Self, Self::Error> {
431        let date =
432            NaiveDate::parse_from_str(raw.date, CHRONO_DATE_FMT).map_err(|_e| "date invalid")?;
433        let time =
434            NaiveTime::parse_from_str(raw.time, CHRONO_TIME_FMT).map_err(|_e| "time invalid")?;
435        let datetime = NaiveDateTime::new(date, time);
436
437        let line = Self {
438            date,
439            time: raw.time.to_string(),
440            datetime,
441            x_edge_location: raw.x_edge_location.to_string(),
442            sc_bytes: raw
443                .sc_bytes
444                .parse::<u64>()
445                .map_err(|_e| "sc_bytes invalid")?,
446            c_ip: raw.c_ip.to_string(),
447            cs_method: raw.cs_method.to_string(),
448            cs_host: raw.cs_host.to_string(),
449            cs_uri_stem: raw.cs_uri_stem.to_string(),
450            sc_status: raw
451                .sc_status
452                .parse::<u16>()
453                .map_err(|_e| "sc_status invalid")?,
454            cs_referer: raw.cs_referer.to_optional_string(),
455            cs_user_agent: raw.cs_user_agent.to_string(),
456            cs_uri_query: raw.cs_uri_query.to_optional_string(),
457            cs_cookie: raw.cs_cookie.to_optional_string(),
458            x_edge_result_type: raw.x_edge_result_type.to_string(),
459            x_edge_request_id: raw.x_edge_request_id.to_string(),
460            x_host_header: raw.x_host_header.to_string(),
461            cs_protocol: raw.cs_protocol.to_string(),
462            cs_bytes: raw
463                .cs_bytes
464                .parse::<u64>()
465                .map_err(|_e| "cs_bytes invalid")?,
466            time_taken: raw
467                .time_taken
468                .parse::<f64>()
469                .map_err(|_e| "time_taken invalid")?,
470            x_forwarded_for: raw.x_forwarded_for.to_optional_string(),
471            ssl_protocol: raw.ssl_protocol.to_optional_string(),
472            ssl_cipher: raw.ssl_cipher.to_optional_string(),
473            x_edge_response_result_type: raw.x_edge_response_result_type.to_string(),
474            cs_protocol_version: raw.cs_protocol_version.to_string(),
475            fle_status: raw.fle_status.to_optional_string(),
476            fle_encrypted_fields: parse_as_option(raw.fle_encrypted_fields)
477                .map_err(|_e| "fle_encrypted_fields invalid")?,
478            c_port: raw.c_port.parse::<u16>().map_err(|_e| "c_port invalid")?,
479            time_to_first_byte: raw
480                .time_to_first_byte
481                .parse::<f64>()
482                .map_err(|_e| "time_to_first_byte invalid")?,
483            x_edge_detailed_result_type: raw.x_edge_detailed_result_type.to_string(),
484            sc_content_type: raw.sc_content_type.to_optional_string(),
485            sc_content_len: parse_as_option(raw.sc_content_len)
486                .map_err(|_e| "sc_content_len invalid")?,
487            sc_range_start: parse_as_option(raw.sc_range_start)
488                .map_err(|_e| "sc_range_start invalid")?,
489            sc_range_end: parse_as_option(raw.sc_range_end).map_err(|_e| "sc_range_end invalid")?,
490        };
491        Ok(line)
492    }
493}
494
495impl TryFrom<ValidatedRaw<'_>> for UnvalidatedLogline {
496    type Error = &'static str;
497
498    fn try_from(raw: ValidatedRaw<'_>) -> Result<Self, Self::Error> {
499        let date =
500            NaiveDate::parse_from_str(raw.date, CHRONO_DATE_FMT).map_err(|_e| "date invalid")?;
501        let time =
502            NaiveTime::parse_from_str(raw.time, CHRONO_TIME_FMT).map_err(|_e| "time invalid")?;
503        let datetime = NaiveDateTime::new(date, time);
504
505        let line = Self {
506            date,
507            time: raw.time.to_string(),
508            datetime,
509            x_edge_location: raw.x_edge_location.to_string(),
510            sc_bytes: raw
511                .sc_bytes
512                .parse::<u64>()
513                .map_err(|_e| "sc_bytes invalid")?,
514            c_ip: raw.c_ip.to_string(),
515            cs_method: raw.cs_method.to_string(),
516            cs_host: raw.cs_host.to_string(),
517            cs_uri_stem: raw.cs_uri_stem.to_string(),
518            sc_status: raw
519                .sc_status
520                .parse::<u16>()
521                .map_err(|_e| "sc_status invalid")?,
522            cs_referer: raw.cs_referer.to_optional_string(),
523            cs_user_agent: raw.cs_user_agent.to_string(),
524            cs_uri_query: raw.cs_uri_query.to_optional_string(),
525            cs_cookie: raw.cs_cookie.to_optional_string(),
526            x_edge_result_type: raw.x_edge_result_type.to_string(),
527            x_edge_request_id: raw.x_edge_request_id.to_string(),
528            x_host_header: raw.x_host_header.to_string(),
529            cs_protocol: raw.cs_protocol.to_string(),
530            cs_bytes: raw
531                .cs_bytes
532                .parse::<u64>()
533                .map_err(|_e| "cs_bytes invalid")?,
534            time_taken: raw
535                .time_taken
536                .parse::<f64>()
537                .map_err(|_e| "time_taken invalid")?,
538            x_forwarded_for: raw.x_forwarded_for.to_optional_string(),
539            ssl_protocol: raw.ssl_protocol.to_optional_string(),
540            ssl_cipher: raw.ssl_cipher.to_optional_string(),
541            x_edge_response_result_type: raw.x_edge_response_result_type.to_string(),
542            cs_protocol_version: raw.cs_protocol_version.to_string(),
543            fle_status: raw.fle_status.to_optional_string(),
544            fle_encrypted_fields: parse_as_option(raw.fle_encrypted_fields)
545                .map_err(|_e| "fle_encrypted_fields invalid")?,
546            c_port: raw.c_port.parse::<u16>().map_err(|_e| "c_port invalid")?,
547            time_to_first_byte: raw
548                .time_to_first_byte
549                .parse::<f64>()
550                .map_err(|_e| "time_to_first_byte invalid")?,
551            x_edge_detailed_result_type: raw.x_edge_detailed_result_type.to_string(),
552            sc_content_type: raw.sc_content_type.to_optional_string(),
553            sc_content_len: parse_as_option(raw.sc_content_len)
554                .map_err(|_e| "sc_content_len invalid")?,
555            sc_range_start: parse_as_option(raw.sc_range_start)
556                .map_err(|_e| "sc_range_start invalid")?,
557            sc_range_end: parse_as_option(raw.sc_range_end).map_err(|_e| "sc_range_end invalid")?,
558        };
559        Ok(line)
560    }
561}
562
563impl From<ValidatedLogline> for UnvalidatedLogline {
564    fn from(validated: ValidatedLogline) -> Self {
565        UnvalidatedLogline {
566            date: validated.date,
567            time: validated.time,
568            datetime: validated.datetime,
569            x_edge_location: validated.x_edge_location,
570            sc_bytes: validated.sc_bytes,
571            c_ip: validated.c_ip,
572            cs_method: validated.cs_method,
573            cs_host: validated.cs_host,
574            cs_uri_stem: validated.cs_uri_stem,
575            sc_status: validated.sc_status,
576            cs_referer: validated.cs_referer,
577            cs_user_agent: validated.cs_user_agent,
578            cs_uri_query: validated.cs_uri_query,
579            cs_cookie: validated.cs_cookie,
580            x_edge_result_type: validated.x_edge_result_type,
581            x_edge_request_id: validated.x_edge_request_id,
582            x_host_header: validated.x_host_header,
583            cs_protocol: validated.cs_protocol,
584            cs_bytes: validated.cs_bytes,
585            time_taken: validated.time_taken,
586            x_forwarded_for: validated.x_forwarded_for,
587            ssl_protocol: validated.ssl_protocol,
588            ssl_cipher: validated.ssl_cipher,
589            x_edge_response_result_type: validated.x_edge_response_result_type,
590            cs_protocol_version: validated.cs_protocol_version,
591            fle_status: validated.fle_status,
592            fle_encrypted_fields: validated.fle_encrypted_fields,
593            c_port: validated.c_port,
594            time_to_first_byte: validated.time_to_first_byte,
595            x_edge_detailed_result_type: validated.x_edge_detailed_result_type,
596            sc_content_type: validated.sc_content_type,
597            sc_content_len: validated.sc_content_len,
598            sc_range_start: validated.sc_range_start,
599            sc_range_end: validated.sc_range_end,
600        }
601    }
602}
603
604impl From<UnvalidatedLogline> for ValidatedLogline {
605    fn from(unvalidated: UnvalidatedLogline) -> Self {
606        ValidatedLogline {
607            date: unvalidated.date,
608            time: unvalidated.time,
609            datetime: unvalidated.datetime,
610            x_edge_location: unvalidated.x_edge_location,
611            sc_bytes: unvalidated.sc_bytes,
612            c_ip: unvalidated.c_ip,
613            cs_method: unvalidated.cs_method,
614            cs_host: unvalidated.cs_host,
615            cs_uri_stem: unvalidated.cs_uri_stem,
616            sc_status: unvalidated.sc_status,
617            cs_referer: unvalidated.cs_referer,
618            cs_user_agent: unvalidated.cs_user_agent,
619            cs_uri_query: unvalidated.cs_uri_query,
620            cs_cookie: unvalidated.cs_cookie,
621            x_edge_result_type: unvalidated.x_edge_result_type,
622            x_edge_request_id: unvalidated.x_edge_request_id,
623            x_host_header: unvalidated.x_host_header,
624            cs_protocol: unvalidated.cs_protocol,
625            cs_bytes: unvalidated.cs_bytes,
626            time_taken: unvalidated.time_taken,
627            x_forwarded_for: unvalidated.x_forwarded_for,
628            ssl_protocol: unvalidated.ssl_protocol,
629            ssl_cipher: unvalidated.ssl_cipher,
630            x_edge_response_result_type: unvalidated.x_edge_response_result_type,
631            cs_protocol_version: unvalidated.cs_protocol_version,
632            fle_status: unvalidated.fle_status,
633            fle_encrypted_fields: unvalidated.fle_encrypted_fields,
634            c_port: unvalidated.c_port,
635            time_to_first_byte: unvalidated.time_to_first_byte,
636            x_edge_detailed_result_type: unvalidated.x_edge_detailed_result_type,
637            sc_content_type: unvalidated.sc_content_type,
638            sc_content_len: unvalidated.sc_content_len,
639            sc_range_start: unvalidated.sc_range_start,
640            sc_range_end: unvalidated.sc_range_end,
641        }
642    }
643}
cloudfront_logs/owned/parquet.rs

cloudfront_logs/owned/
parquet.rs
