sas7bdat 0.2.0

Rust library + CLI for decoding SAS7BDAT datasets and streaming them to modern formats.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

use super::constants::UTF8_DICTIONARY_LIMIT;
use ahash::RandomState;
use bytes::Bytes;
use hashbrown::{HashMap, hash_map::RawEntryMut};
use parquet::data_type::ByteArray;

pub(super) struct Utf8Scratch {
    pub ryu: ryu::Buffer,
    pub itoa: itoa::Buffer,
    dictionary: HashMap<Vec<u8>, ByteArray, RandomState>,
    dictionary_enabled: bool,
    last_short: Option<(Vec<u8>, ByteArray)>,
}

impl Utf8Scratch {
    pub(crate) fn new() -> Self {
        Self {
            ryu: ryu::Buffer::new(),
            itoa: itoa::Buffer::new(),
            dictionary: HashMap::with_capacity_and_hasher(
                UTF8_DICTIONARY_LIMIT,
                RandomState::new(),
            ),
            dictionary_enabled: true,
            last_short: None,
        }
    }

    pub(crate) fn intern_slice(&mut self, data: &[u8]) -> ByteArray {
        if data.len() <= 32
            && let Some((ref previous, ref handle)) = self.last_short
            && previous.as_slice() == data
        {
            return handle.clone();
        }
        if self.dictionary_enabled && self.dictionary.len() >= UTF8_DICTIONARY_LIMIT {
            self.dictionary.clear();
            self.dictionary_enabled = false;
        }
        if !self.dictionary_enabled {
            let stored = ByteArray::from(Bytes::copy_from_slice(data));
            if data.len() <= 32 {
                self.last_short = Some((data.to_vec(), stored.clone()));
            }
            return stored;
        }
        match self.dictionary.raw_entry_mut().from_key(data) {
            RawEntryMut::Occupied(entry) => {
                let cloned = entry.get().clone();
                if data.len() <= 32 {
                    self.last_short = Some((data.to_vec(), cloned.clone()));
                }
                cloned
            }
            RawEntryMut::Vacant(vacant) => {
                let stored = ByteArray::from(Bytes::copy_from_slice(data));
                vacant.insert(data.to_vec(), stored.clone());
                if data.len() <= 32 {
                    self.last_short = Some((data.to_vec(), stored.clone()));
                }
                stored
            }
        }
    }

    pub(crate) fn intern_str(&mut self, text: &str) -> ByteArray {
        self.intern_slice(text.as_bytes())
    }
}