dmap/
record.rs

1//! Defines the [`Record`] trait, which contains the shared behaviour that all DMAP records must have.
2
3use crate::convenience::split_results;
4use crate::error::DmapError;
5use crate::io::{self};
6use crate::io::{create_stream, slice_stream_lax, split_into_slices};
7use crate::types::{DmapField, DmapType, DmapVec, Fields};
8use indexmap::IndexMap;
9use itertools::izip;
10use rayon::iter::Either;
11use rayon::prelude::*;
12use std::fmt::Debug;
13use std::fs::File;
14use std::io::Read;
15use std::path::Path;
16
17/// DMAP record template.
18///
19/// This trait defines functionality for parsing bytes into records, converting records to bytes,
20/// and reading from / writing to files.
21pub trait Record<'a>:
22    Debug + Send + Sync + TryFrom<IndexMap<String, DmapField>, Error = DmapError>
23{
24    /// Creates a new object from the parsed scalars and vectors.
25    fn new(fields: &mut IndexMap<String, DmapField>) -> Result<Self, DmapError>
26    where
27        Self: Sized;
28
29    /// Gets the underlying data of `self`.
30    fn inner(self) -> IndexMap<String, DmapField>;
31
32    /// Returns the field with name `key`, if it exists in the record.
33    fn get(&self, key: &str) -> Option<&DmapField>;
34
35    /// Returns the names of all fields stored in the record.
36    fn keys(&self) -> Vec<&String>;
37
38    /// Returns whether `name` is a metadata field of the record.
39    fn is_metadata_field(name: &str) -> bool;
40
41    /// Reads the nth records from `dmap_data` and parse into instances of `Self`.
42    ///
43    /// Returns `DmapError` if `dmap_data` cannot be read or contains invalid data.
44    fn read_nth_records(dmap_data: impl Read, indices: &[i32]) -> Result<Vec<Self>, DmapError>
45    where
46        Self: Sized,
47        Self: Send,
48    {
49        let mut slices = split_into_slices(dmap_data)?;
50        let num_recs = slices.len();
51        let mut records_read = vec![];
52        for idx in indices.iter() {
53            if *idx >= num_recs as i32 || *idx <= -1 * num_recs as i32 {
54                return Err(DmapError::InvalidIndex(*idx));
55            }
56            let i = {
57                if *idx < 0 {
58                    num_recs - idx.abs() as usize
59                } else {
60                    *idx as usize
61                }
62            };
63            records_read.push(slices[i].parse_record::<Self>().map_err(|e| {
64                DmapError::InvalidRecord(format!("Record {idx}: {}", e.to_string()))
65            })?);
66        }
67        Ok(records_read)
68    }
69
70    /// Reads the nth records from `dmap_data` and parse into instances of `Self`, if possible.
71    ///
72    /// Returns a 2-tuple, where the first entry is the good records from the front of the buffer,
73    /// and the second entry is the byte where the first corrupted record starts, if applicable.
74    fn read_nth_records_lax(
75        mut dmap_data: impl Read,
76        indices: &[i32],
77    ) -> Result<(Vec<Self>, Option<usize>), DmapError>
78    where
79        Self: Sized,
80        Self: Send,
81    {
82        let mut buffer: Vec<u8> = vec![];
83        let mut dmap_records: Vec<Self> = vec![];
84
85        create_stream(&mut dmap_data)?.read_to_end(&mut buffer)?;
86        let (mut slices, rec_starts, mut bad_byte) = slice_stream_lax(buffer);
87
88        let num_recs = slices.len();
89        for idx in indices.iter() {
90            if *idx >= num_recs as i32 || *idx <= -1 * num_recs as i32 {
91                return Err(DmapError::InvalidIndex(*idx));
92            }
93            let i = {
94                if *idx < 0 {
95                    num_recs - idx.abs() as usize
96                } else {
97                    *idx as usize
98                }
99            };
100            let parse_result = slices[i].parse_record::<Self>();
101            if let Ok(x) = parse_result {
102                dmap_records.push(x);
103            } else {
104                bad_byte = Some(rec_starts[i]);
105                break;
106            }
107        }
108
109        Ok((dmap_records, bad_byte))
110    }
111
112    /// Reads from `dmap_data` and parses into `Vec<Self>`.
113    ///
114    /// Returns `DmapError` if `dmap_data` cannot be read or contains invalid data.
115    fn read_records(dmap_data: impl Read) -> Result<Vec<Self>, DmapError>
116    where
117        Self: Sized,
118        Self: Send,
119    {
120        let mut slices = split_into_slices(dmap_data)?;
121        let mut dmap_results: Vec<Result<Self, DmapError>> = vec![];
122        dmap_results.par_extend(
123            slices
124                .par_iter_mut()
125                .map(|parser| parser.parse_record::<Self>()),
126        );
127
128        let (dmap_records, dmap_errors, bad_recs) = split_results(dmap_results);
129        if !dmap_errors.is_empty() {
130            return Err(DmapError::BadRecords(bad_recs, dmap_errors[0].to_string()));
131        }
132        Ok(dmap_records)
133    }
134
135    /// Reads metadata of records from `dmap_data` and parses into `Vec<IndexMap<String, DmapField>>`.
136    ///
137    /// Returns `DmapError` if `dmap_data` cannot be read or contains invalid data.
138    fn read_metadata(dmap_data: impl Read) -> Result<Vec<IndexMap<String, DmapField>>, DmapError>
139    where
140        Self: Sized,
141        Self: Send,
142    {
143        let mut slices = split_into_slices(dmap_data)?;
144        let mut dmap_results: Vec<Result<IndexMap<String, DmapField>, DmapError>> = vec![];
145        dmap_results.par_extend(
146            slices
147                .par_iter_mut()
148                .map(|parser| parser.parse_metadata::<Self>()),
149        );
150
151        let (dmap_records, dmap_errors, bad_recs) = split_results(dmap_results);
152        if !dmap_errors.is_empty() {
153            return Err(DmapError::BadRecords(bad_recs, dmap_errors[0].to_string()));
154        }
155
156        Ok(dmap_records)
157    }
158
159    /// Reads metadata of the nth records from `dmap_data` and parses into `Vec<IndexMap<String, DmapField>>`.
160    ///
161    /// Returns `DmapError` if `dmap_data` cannot be read or contains invalid data.
162    fn read_metadata_by_indices(
163        dmap_data: impl Read,
164        indices: &[i32],
165    ) -> Result<Vec<IndexMap<String, DmapField>>, DmapError>
166    where
167        Self: Sized,
168        Self: Send,
169    {
170        let mut slices = split_into_slices(dmap_data)?;
171        let mut dmap_results: Vec<IndexMap<String, DmapField>> = vec![];
172
173        let num_recs = slices.len();
174        for idx in indices.iter() {
175            if *idx >= num_recs as i32 || *idx <= -1 * num_recs as i32 {
176                return Err(DmapError::InvalidIndex(*idx));
177            }
178            let i = {
179                if *idx < 0 {
180                    num_recs - idx.abs() as usize
181                } else {
182                    *idx as usize
183                }
184            };
185            dmap_results.push(slices[i].parse_metadata::<Self>().map_err(|e| {
186                DmapError::InvalidRecord(format!("Record {idx}: {}", e.to_string()))
187            })?);
188        }
189        Ok(dmap_results)
190    }
191
192    /// Reads from `dmap_data` and parses into `Vec<Self>`.
193    ///
194    /// Returns a 2-tuple, where the first entry is the good records from the front of the buffer,
195    /// and the second entry is the byte where the first corrupted record starts, if applicable.
196    fn read_records_lax(mut dmap_data: impl Read) -> Result<(Vec<Self>, Option<usize>), DmapError>
197    where
198        Self: Sized,
199        Self: Send,
200    {
201        let mut buffer: Vec<u8> = vec![];
202        let mut dmap_records: Vec<Self> = vec![];
203
204        create_stream(&mut dmap_data)?.read_to_end(&mut buffer)?;
205        let (mut slices, rec_starts, mut bad_byte) = slice_stream_lax(buffer);
206
207        let mut dmap_results: Vec<Result<Self, DmapError>> = vec![];
208        dmap_results.par_extend(
209            slices
210                .par_iter_mut()
211                .map(|parser| parser.parse_record::<Self>()),
212        );
213
214        for (i, rec) in dmap_results.into_iter().enumerate() {
215            if let Ok(x) = rec {
216                dmap_records.push(x);
217            } else {
218                bad_byte = Some(rec_starts[i]);
219                break;
220            }
221        }
222        Ok((dmap_records, bad_byte))
223    }
224
225    /// Read a DMAP file of type `Self`
226    fn read_file<P: AsRef<Path>>(infile: P) -> Result<Vec<Self>, DmapError>
227    where
228        Self: Sized,
229        Self: Send,
230    {
231        let file = File::open(infile)?;
232        Self::read_records(file)
233    }
234
235    /// Read a DMAP file of type `Self`.
236    ///
237    /// If the file is corrupted, it will return the leading uncorrupted records as well as the
238    /// position corresponding to the start of the first corrupted record.
239    fn read_file_lax<P: AsRef<Path>>(infile: P) -> Result<(Vec<Self>, Option<usize>), DmapError>
240    where
241        Self: Sized,
242        Self: Send,
243    {
244        let file = File::open(infile)?;
245        Self::read_records_lax(file)
246    }
247
248    /// Reads the `nth` record(s) of a DMAP file of type `Self`.
249    fn read_file_by_indices<P: AsRef<Path>>(
250        infile: P,
251        indices: &[i32],
252    ) -> Result<Vec<Self>, DmapError>
253    where
254        Self: Sized,
255        Self: Send,
256    {
257        let file = File::open(infile)?;
258        Self::read_nth_records(file, indices)
259    }
260
261    /// Reads the `nth` record(s) of a DMAP file of type `Self`.
262    ///
263    /// Does not fail on corrupted records; rather, returns `(recs, Option<usize>)` where
264    /// the second value is the byte where record corruption begins, if applicable.
265    fn read_file_by_indices_lax<P: AsRef<Path>>(
266        infile: P,
267        indices: &[i32],
268    ) -> Result<(Vec<Self>, Option<usize>), DmapError>
269    where
270        Self: Sized,
271        Self: Send,
272    {
273        let file = File::open(infile)?;
274        Self::read_nth_records_lax(file, indices)
275    }
276
277    /// Read the metadata from a DMAP file of type `Self`
278    fn read_file_metadata<P: AsRef<Path>>(
279        infile: P,
280    ) -> Result<Vec<IndexMap<String, DmapField>>, DmapError>
281    where
282        Self: Sized,
283        Self: Send,
284    {
285        let file = File::open(infile)?;
286        Self::read_metadata(file)
287    }
288
289    /// Reads the `nth` records' metadata of a DMAP file of type `Self`.
290    fn read_file_metadata_by_indices<P: AsRef<Path>>(
291        infile: P,
292        indices: &[i32],
293    ) -> Result<Vec<IndexMap<String, DmapField>>, DmapError>
294    where
295        Self: Sized,
296        Self: Send,
297    {
298        let file = File::open(infile)?;
299        Self::read_metadata_by_indices(file, indices)
300    }
301
302    /// Checks the validity of an `IndexMap` as a representation of a DMAP record.
303    ///
304    /// Validity checks include ensuring that no unfamiliar entries exist, that all required
305    /// scalar and vector fields exist, that all scalar and vector fields are of the expected
306    /// type, and that vector fields which are expected to have the same dimensions do indeed
307    /// have the same dimensions.
308    fn check_fields(
309        field_dict: &mut IndexMap<String, DmapField>,
310        fields_for_type: &Fields,
311    ) -> Result<(), DmapError> {
312        let unsupported_keys: Vec<&String> = field_dict
313            .keys()
314            .filter(|&k| !fields_for_type.all_fields.contains(&&**k))
315            .collect();
316        if !unsupported_keys.is_empty() {
317            Err(DmapError::InvalidRecord(format!(
318                "Unsupported fields {:?}, fields supported are {:?}",
319                unsupported_keys, fields_for_type.all_fields
320            )))?
321        }
322
323        for (field, expected_type) in fields_for_type.scalars_required.iter() {
324            match field_dict.get(&field.to_string()) {
325                Some(DmapField::Scalar(x)) if &x.get_type() == expected_type => {}
326                Some(DmapField::Scalar(x)) => Err(DmapError::InvalidRecord(format!(
327                    "Field {} has incorrect type {}, expected {}",
328                    field,
329                    x.get_type(),
330                    expected_type
331                )))?,
332                Some(_) => Err(DmapError::InvalidRecord(format!(
333                    "Field {} is a vector, expected scalar",
334                    field
335                )))?,
336                None => Err(DmapError::InvalidRecord(format!(
337                    "Field {field:?} ({:?}) missing: fields {:?}",
338                    &field.to_string(),
339                    field_dict.keys()
340                )))?,
341            }
342        }
343        for (field, expected_type) in fields_for_type.scalars_optional.iter() {
344            match field_dict.get(&field.to_string()) {
345                Some(DmapField::Scalar(x)) if &x.get_type() == expected_type => {}
346                Some(DmapField::Scalar(x)) => Err(DmapError::InvalidRecord(format!(
347                    "Field {} has incorrect type {}, expected {}",
348                    field,
349                    x.get_type(),
350                    expected_type
351                )))?,
352                Some(_) => Err(DmapError::InvalidRecord(format!(
353                    "Field {} is a vector, expected scalar",
354                    field
355                )))?,
356                None => {}
357            }
358        }
359        for (field, expected_type) in fields_for_type.vectors_required.iter() {
360            match field_dict.get(&field.to_string()) {
361                Some(DmapField::Scalar(_)) => Err(DmapError::InvalidRecord(format!(
362                    "Field {} is a scalar, expected vector",
363                    field
364                )))?,
365                Some(DmapField::Vector(x)) if &x.get_type() != expected_type => {
366                    Err(DmapError::InvalidRecord(format!(
367                        "Field {field} has incorrect type {:?}, expected {expected_type:?}",
368                        x.get_type()
369                    )))?
370                }
371                Some(&DmapField::Vector(_)) => {}
372                None => Err(DmapError::InvalidRecord(format!("Field {field} missing")))?,
373            }
374        }
375        for (field, expected_type) in fields_for_type.vectors_optional.iter() {
376            match field_dict.get(&field.to_string()) {
377                Some(&DmapField::Scalar(_)) => Err(DmapError::InvalidRecord(format!(
378                    "Field {} is a scalar, expected vector",
379                    field
380                )))?,
381                Some(DmapField::Vector(x)) if &x.get_type() != expected_type => {
382                    Err(DmapError::InvalidRecord(format!(
383                        "Field {field} has incorrect type {}, expected {expected_type}",
384                        x.get_type()
385                    )))?
386                }
387                _ => {}
388            }
389        }
390        // This block checks that grouped vector fields have the same dimensionality
391        for vec_group in fields_for_type.vector_dim_groups.iter() {
392            let vecs: Vec<(&str, &DmapVec)> = vec_group
393                .iter()
394                .filter_map(|&name| match field_dict.get(&name.to_string()) {
395                    Some(DmapField::Vector(ref x)) => Some((name, x)),
396                    Some(_) => None,
397                    None => None,
398                })
399                .collect();
400            if vecs.len() > 1 {
401                let mut vec_iter = vecs.iter();
402                let first = vec_iter.next().expect("Iterator broken");
403                if !vec_iter.all(|(_, v)| v.shape() == first.1.shape()) {
404                    let error_vec: Vec<(&str, &[usize])> =
405                        vecs.iter().map(|(k, v)| (*k, v.shape())).collect();
406                    Err(DmapError::InvalidRecord(format!(
407                        "Vector fields have inconsistent dimensions: {:?}",
408                        error_vec
409                    )))?
410                }
411            }
412        }
413        Ok(())
414    }
415
416    /// Attempts to massage the entries of an `IndexMap` into the proper types for a DMAP record.
417    fn coerce(
418        fields_dict: &mut IndexMap<String, DmapField>,
419        fields_for_type: &Fields,
420    ) -> Result<Self, DmapError> {
421        let unsupported_keys: Vec<&String> = fields_dict
422            .keys()
423            .filter(|&k| !fields_for_type.all_fields.contains(&&**k))
424            .collect();
425        if !unsupported_keys.is_empty() {
426            Err(DmapError::InvalidRecord(format!(
427                "Unsupported fields {:?}, fields supported are {:?}",
428                unsupported_keys, fields_for_type.all_fields
429            )))?
430        }
431
432        for (field, expected_type) in fields_for_type.scalars_required.iter() {
433            match fields_dict.get(&field.to_string()) {
434                Some(DmapField::Scalar(x)) if &x.get_type() != expected_type => {
435                    fields_dict.insert(
436                        field.to_string(),
437                        DmapField::Scalar(x.cast_as(expected_type)?),
438                    );
439                }
440                Some(DmapField::Scalar(_)) => {}
441                Some(_) => Err(DmapError::InvalidRecord(format!(
442                    "Field {} is a vector, expected scalar",
443                    field
444                )))?,
445                None => Err(DmapError::InvalidRecord(format!(
446                    "Field {field:?} ({:?}) missing: fields {:?}",
447                    &field.to_string(),
448                    fields_dict.keys()
449                )))?,
450            }
451        }
452        for (field, expected_type) in fields_for_type.scalars_optional.iter() {
453            match fields_dict.get(&field.to_string()) {
454                Some(DmapField::Scalar(x)) if &x.get_type() == expected_type => {}
455                Some(DmapField::Scalar(x)) => {
456                    fields_dict.insert(
457                        field.to_string(),
458                        DmapField::Scalar(x.cast_as(expected_type)?),
459                    );
460                }
461                Some(_) => Err(DmapError::InvalidRecord(format!(
462                    "Field {} is a vector, expected scalar",
463                    field
464                )))?,
465                None => {}
466            }
467        }
468        for (field, expected_type) in fields_for_type.vectors_required.iter() {
469            match fields_dict.get(&field.to_string()) {
470                Some(DmapField::Scalar(_)) => Err(DmapError::InvalidRecord(format!(
471                    "Field {} is a scalar, expected vector",
472                    field
473                )))?,
474                Some(DmapField::Vector(x)) if &x.get_type() != expected_type => {
475                    Err(DmapError::InvalidRecord(format!(
476                        "Field {field} has incorrect type {:?}, expected {expected_type:?}",
477                        x.get_type()
478                    )))?
479                }
480                Some(DmapField::Vector(_)) => {}
481                None => Err(DmapError::InvalidRecord(format!("Field {field} missing")))?,
482            }
483        }
484        for (field, expected_type) in fields_for_type.vectors_optional.iter() {
485            match fields_dict.get(&field.to_string()) {
486                Some(&DmapField::Scalar(_)) => Err(DmapError::InvalidRecord(format!(
487                    "Field {} is a scalar, expected vector",
488                    field
489                )))?,
490                Some(DmapField::Vector(x)) if &x.get_type() != expected_type => {
491                    Err(DmapError::InvalidRecord(format!(
492                        "Field {field} has incorrect type {}, expected {expected_type}",
493                        x.get_type()
494                    )))?
495                }
496                _ => {}
497            }
498        }
499
500        Self::new(fields_dict)
501    }
502
503    /// Attempts to copy `self` to a raw byte representation.
504    fn to_bytes(&self) -> Result<Vec<u8>, DmapError>;
505
506    /// Converts the entries of an `IndexMap` into a raw byte representation, including metadata
507    /// about the entries `(DMAP key, name\[, dimensions\])`.
508    ///
509    /// If all is good, returns a tuple containing:
510    /// * the number of scalar fields
511    /// * the number of vector fields
512    /// * the raw bytes
513    fn data_to_bytes(
514        data: &IndexMap<String, DmapField>,
515        fields_for_type: &Fields,
516    ) -> Result<(i32, i32, Vec<u8>), DmapError> {
517        let mut data_bytes: Vec<u8> = vec![];
518        let mut num_scalars: i32 = 0;
519        let mut num_vectors: i32 = 0;
520
521        // let scalar_fields = data.keys().filter(|k| )
522        for (field, _) in fields_for_type.scalars_required.iter() {
523            match data.get(&field.to_string()) {
524                Some(x @ DmapField::Scalar(_)) => {
525                    data_bytes.extend(field.as_bytes());
526                    data_bytes.extend([0]); // null-terminate string
527                    data_bytes.append(&mut x.as_bytes());
528                    num_scalars += 1;
529                }
530                Some(_) => Err(DmapError::InvalidScalar(format!(
531                    "Field {field} is a vector, expected scalar"
532                )))?,
533                None => Err(DmapError::InvalidRecord(format!(
534                    "Field {field} missing from record"
535                )))?,
536            }
537        }
538        for (field, _) in fields_for_type.scalars_optional.iter() {
539            if let Some(x) = data.get(&field.to_string()) {
540                match x {
541                    DmapField::Scalar(_) => {
542                        data_bytes.extend(field.as_bytes());
543                        data_bytes.extend([0]); // null-terminate string
544                        data_bytes.append(&mut x.as_bytes());
545                        num_scalars += 1;
546                    }
547                    DmapField::Vector(_) => Err(DmapError::InvalidScalar(format!(
548                        "Field {field} is a vector, expected scalar"
549                    )))?,
550                }
551            }
552        }
553        for (field, _) in fields_for_type.vectors_required.iter() {
554            match data.get(&field.to_string()) {
555                Some(x @ DmapField::Vector(_)) => {
556                    data_bytes.extend(field.as_bytes());
557                    data_bytes.extend([0]); // null-terminate string
558                    data_bytes.append(&mut x.as_bytes());
559                    num_vectors += 1;
560                }
561                Some(_) => Err(DmapError::InvalidVector(format!(
562                    "Field {field} is a scalar, expected vector"
563                )))?,
564                None => Err(DmapError::InvalidRecord(format!(
565                    "Field {field} missing from record"
566                )))?,
567            }
568        }
569        for (field, _) in fields_for_type.vectors_optional.iter() {
570            if let Some(x) = data.get(&field.to_string()) {
571                match x {
572                    DmapField::Vector(_) => {
573                        data_bytes.extend(field.as_bytes());
574                        data_bytes.extend([0]); // null-terminate string
575                        data_bytes.append(&mut x.as_bytes());
576                        num_vectors += 1;
577                    }
578                    DmapField::Scalar(_) => Err(DmapError::InvalidVector(format!(
579                        "Field {field} is a scalar, expected vector"
580                    )))?,
581                }
582            }
583        }
584
585        Ok((num_scalars, num_vectors, data_bytes))
586    }
587
588    /// Converts the entries of a `Record` into a raw byte representation, for debugging the conversion.
589    ///
590    /// If all is good, returns a vector containing tuples of:
591    /// * `String`: the name of the field (`"header"` denoting the record header)
592    /// * `usize`: where the serialized bytes of the field start in the record byte representation
593    /// * `Vec<u8>` the byte representation of the field.
594    fn inspect_bytes(
595        &self,
596        fields_for_type: &Fields,
597    ) -> Result<Vec<(String, usize, Vec<u8>)>, DmapError> {
598        let mut data_bytes: Vec<Vec<u8>> = vec![];
599        let mut indices: Vec<usize> = vec![16]; // start at 16 to account for header
600        let mut fields: Vec<String> = vec![];
601
602        let (mut num_scalars, mut num_vectors) = (0, 0);
603
604        for (field, _) in fields_for_type.scalars_required.iter() {
605            fields.push(field.to_string());
606            match self.get(field) {
607                Some(x @ DmapField::Scalar(_)) => {
608                    let mut bytes = vec![];
609                    bytes.extend(field.as_bytes());
610                    bytes.extend([0]); // null-terminate string
611                    bytes.append(&mut x.as_bytes());
612                    indices.push(indices[indices.len() - 1] + bytes.len());
613                    data_bytes.push(bytes);
614                    num_scalars += 1;
615                }
616                Some(_) => Err(DmapError::InvalidScalar(format!(
617                    "Field {field} is a vector, expected scalar"
618                )))?,
619                None => Err(DmapError::InvalidRecord(format!(
620                    "Field {field} missing from record"
621                )))?,
622            }
623        }
624        for (field, _) in fields_for_type.scalars_optional.iter() {
625            fields.push(field.to_string());
626            if let Some(x) = self.get(field) {
627                match x {
628                    DmapField::Scalar(_) => {
629                        let mut bytes = vec![];
630                        bytes.extend(field.as_bytes());
631                        bytes.extend([0]); // null-terminate string
632                        bytes.append(&mut x.as_bytes());
633                        indices.push(indices[indices.len() - 1] + bytes.len());
634                        data_bytes.push(bytes);
635                        num_scalars += 1;
636                    }
637                    DmapField::Vector(_) => Err(DmapError::InvalidScalar(format!(
638                        "Field {field} is a vector, expected scalar"
639                    )))?,
640                }
641            }
642        }
643        for (field, _) in fields_for_type.vectors_required.iter() {
644            fields.push(field.to_string());
645            match self.get(field) {
646                Some(x @ DmapField::Vector(_)) => {
647                    let mut bytes = vec![];
648                    bytes.extend(field.as_bytes());
649                    bytes.extend([0]); // null-terminate string
650                    bytes.append(&mut x.as_bytes());
651                    indices.push(indices[indices.len() - 1] + bytes.len());
652                    data_bytes.push(bytes);
653                    num_vectors += 1;
654                }
655                Some(_) => Err(DmapError::InvalidVector(format!(
656                    "Field {field} is a scalar, expected vector"
657                )))?,
658                None => Err(DmapError::InvalidRecord(format!(
659                    "Field {field} missing from record"
660                )))?,
661            }
662        }
663        for (field, _) in fields_for_type.vectors_optional.iter() {
664            fields.push(field.to_string());
665            if let Some(x) = self.get(field) {
666                match x {
667                    DmapField::Vector(_) => {
668                        let mut bytes = vec![];
669                        bytes.extend(field.as_bytes());
670                        bytes.extend([0]); // null-terminate string
671                        bytes.append(&mut x.as_bytes());
672                        indices.push(indices[indices.len() - 1] + data_bytes.len());
673                        data_bytes.push(bytes);
674                        num_vectors += 1;
675                    }
676                    DmapField::Scalar(_) => Err(DmapError::InvalidVector(format!(
677                        "Field {field} is a scalar, expected vector"
678                    )))?,
679                }
680            }
681        }
682
683        // Now build up the header
684        let num_bytes: usize = data_bytes.iter().map(|x| x.len()).sum();
685        let mut bytes: Vec<u8> = vec![];
686        bytes.extend((65537_i32).as_bytes()); // No idea why this is what it is, copied from backscatter
687        bytes.extend((num_bytes as i32 + 16).as_bytes()); // +16 for code, length, num_scalars, num_vectors
688        bytes.extend(num_scalars.as_bytes());
689        bytes.extend(num_vectors.as_bytes());
690
691        // Accumulate all the results into one big `Vec`
692        let mut field_info: Vec<(String, usize, Vec<u8>)> = vec![("header".to_string(), 0, bytes)];
693        for (f, (s, b)) in izip!(
694            fields.into_iter(),
695            izip!(indices[..indices.len() - 1].iter(), data_bytes.into_iter())
696        ) {
697            field_info.push((f, *s, b));
698        }
699
700        Ok(field_info)
701    }
702
703    /// Creates the byte representation of a collection of [`Record`]s.
704    ///
705    /// Ordering of the members is preserved.
706    fn par_to_bytes(recs: &[Self]) -> Result<Vec<u8>, DmapError> {
707        let mut bytes: Vec<u8> = vec![];
708        let (errors, rec_bytes): (Vec<_>, Vec<_>) =
709            recs.par_iter()
710                .enumerate()
711                .partition_map(|(i, rec)| match rec.to_bytes() {
712                    Err(e) => Either::Left((i, e)),
713                    Ok(y) => Either::Right(y),
714                });
715        if !errors.is_empty() {
716            Err(DmapError::InvalidRecord(format!(
717                "Corrupted records: {errors:?}"
718            )))?
719        }
720        bytes.par_extend(rec_bytes.into_par_iter().flatten());
721        Ok(bytes)
722    }
723
724    /// Attempts to convert `recs` to `Self` then convert to bytes.
725    fn try_into_bytes(recs: Vec<IndexMap<String, DmapField>>) -> Result<Vec<u8>, DmapError> {
726        let mut bytes: Vec<u8> = vec![];
727        let (errors, rec_bytes): (Vec<_>, Vec<_>) =
728            recs.into_par_iter()
729                .enumerate()
730                .partition_map(|(i, rec)| match Self::try_from(rec) {
731                    Err(e) => Either::Left((i, e)),
732                    Ok(x) => match x.to_bytes() {
733                        Err(e) => Either::Left((i, e)),
734                        Ok(y) => Either::Right(y),
735                    },
736                });
737        if !errors.is_empty() {
738            Err(DmapError::BadRecords(
739                errors.iter().map(|(i, _)| *i).collect(),
740                errors[0].1.to_string(),
741            ))?
742        }
743        bytes.par_extend(rec_bytes.into_par_iter().flatten());
744        Ok(bytes)
745    }
746
747    /// Writes a collection of `Record`s to `outfile`.
748    ///
749    /// See also [`Record::try_write_to_file`] for writing generic
750    /// `IndexMap<String, DmapField>` records that must first be validated
751    /// against `Self`.
752    fn write_to_file<P: AsRef<Path>>(
753        recs: &Vec<Self>,
754        outfile: P,
755        bz2: bool,
756    ) -> Result<(), DmapError> {
757        let bytes: Vec<u8> = Self::par_to_bytes(recs)?;
758        io::bytes_to_file(bytes, outfile, bz2)?;
759        Ok(())
760    }
761
762    /// Attempts to convert a collection of generic DMAP records to `Self` and
763    /// write them to `outfile`.
764    ///
765    /// This is useful when the records are represented as
766    /// `IndexMap<String, DmapField>` values and need to be validated against the
767    /// requirements of a specific DMAP format before being written.
768    ///
769    /// # Errors
770    ///
771    /// Returns an error if any record cannot be converted to `Self`, or if
772    /// writing to `outfile` fails.
773    fn try_write_to_file<P: AsRef<Path>>(
774        recs: Vec<IndexMap<String, DmapField>>,
775        outfile: P,
776        bz2: bool,
777    ) -> Result<(), DmapError> {
778        let bytes = Self::try_into_bytes(recs)?;
779        io::bytes_to_file(bytes, outfile, bz2)?;
780        Ok(())
781    }
782}
783
784macro_rules! create_record_type {
785    ($format:ident, $fields:ident) => {
786        paste::paste! {
787            use crate::types::{DmapType, DmapField};
788            use crate::error::DmapError;
789            use indexmap::IndexMap;
790            use crate::record::Record;
791
792            #[doc = "Struct containing the checked fields of a single `" $format:upper "` record." ]
793            #[derive(Debug, PartialEq, Clone)]
794            pub struct [< $format:camel Record >] {
795                pub data: IndexMap<String, DmapField>,
796            }
797
798            impl Record<'_> for [< $format:camel Record>] {
799                fn inner(self) -> IndexMap<String, DmapField> {
800                    self.data
801                }
802                fn get(&self, key: &str) -> Option<&DmapField> {
803                    self.data.get(key)
804                }
805                fn keys(&self) -> Vec<&String> {
806                    self.data.keys().collect()
807                }
808                fn new(fields: &mut IndexMap<String, DmapField>) -> Result<[< $format:camel Record>], DmapError> {
809                    match Self::check_fields(fields, &$fields) {
810                        Ok(_) => {}
811                        Err(e) => Err(e)?,
812                    }
813
814                    Ok([< $format:camel Record >] {
815                        data: fields.to_owned(),
816                    })
817                }
818                fn to_bytes(&self) -> Result<Vec<u8>, DmapError> {
819                    let (num_scalars, num_vectors, mut data_bytes) =
820                        Self::data_to_bytes(&self.data, &$fields)?;
821
822                    let mut bytes: Vec<u8> = vec![];
823                    bytes.extend((65537_i32).as_bytes()); // No idea why this is what it is, copied from backscatter
824                    bytes.extend((data_bytes.len() as i32 + 16).as_bytes()); // +16 for code, length, num_scalars, num_vectors
825                    bytes.extend(num_scalars.as_bytes());
826                    bytes.extend(num_vectors.as_bytes());
827                    bytes.append(&mut data_bytes); // consumes data_bytes
828                    Ok(bytes)
829                }
830                fn is_metadata_field(name: &str) -> bool {
831                    !$fields.data_fields.iter().any(|e| e == &name)
832                }
833            }
834
835            impl TryFrom<&mut IndexMap<String, DmapField>> for [< $format:camel Record >] {
836                type Error = DmapError;
837
838                fn try_from(value: &mut IndexMap<String, DmapField>) -> Result<Self, Self::Error> {
839                    Self::coerce(value, &$fields)
840                }
841            }
842
843            impl TryFrom<IndexMap<String, DmapField>> for [< $format:camel Record >] {
844                type Error = DmapError;
845
846                fn try_from(mut value: IndexMap<String, DmapField>) -> Result<Self, Self::Error> {
847                    Self::coerce(&mut value, &$fields)
848                }
849            }
850
851            #[cfg(test)]
852            mod tests {
853                use super::*;
854                use std::path::PathBuf;
855
856                /// Creates a test to ensure that the record is still able to be read, even when missing
857                /// some of the optional fields.
858                #[test]
859                fn test_missing_optional_fields() -> Result<(), DmapError> {
860                    let filename: PathBuf = PathBuf::from(format!("tests/test_files/test.{}", stringify!($format)));
861                    let data = [< $format:camel Record >]::read_file_by_indices(&filename, &[0]).expect("Unable to sniff file");
862                    let recs = data[0].clone().inner();
863
864                    for field in $fields.scalars_optional.iter().chain($fields.vectors_optional.iter()) {
865                        let mut cloned_rec = recs.clone();
866                        let _ = cloned_rec.shift_remove(field.0);
867                        let _ = [< $format:camel Record >]::try_from(&mut cloned_rec)?;
868                    }
869                    Ok(())
870                }
871
872                /// Creates a test to ensure that the record is not able to be read when missing
873                /// some of the required fields.
874                #[test]
875                fn test_missing_required_fields() -> Result<(), DmapError> {
876                    let filename: PathBuf = PathBuf::from(format!("tests/test_files/test.{}", stringify!($format)));
877                    let data = [< $format:camel Record >]::read_file_by_indices(&filename, &[0]).expect("Unable to sniff file");
878                    let recs = data[0].clone().inner();
879
880                    for field in $fields.scalars_required.iter().chain($fields.vectors_required.iter()) {
881                        let mut cloned_rec = recs.clone();
882                        let _ = cloned_rec.shift_remove(field.0);
883                        let res = [< $format:camel Record >]::try_from(&mut cloned_rec);
884                        assert!(res.is_err());
885                    }
886                    Ok(())
887                }
888            }
889        }
890    }
891}
892
893pub(crate) use create_record_type;
dmap/record.rs

dmap/
record.rs
