oak-dsv 0.0.11

High-performance incremental DSV (Delimiter-Separated Values) parser for the oak ecosystem with flexible configuration, optimized for data parsing and tabular data processing.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

use oak_core::{TokenType, UniversalTokenRole};

/// DSV token type
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum DsvTokenType {
    /// Whitespace
    Whitespace,
    /// Newline
    Newline,

    /// Field value (quoted or unquoted)
    Field,
    /// Quoted field value
    QuotedField,
    /// Unquoted field value
    UnquotedField,
    /// Field separator (e.g., , or \t)
    Separator,
    /// Quote character "
    Quote,
    /// Escaped quote ""
    EscapedQuote,

    /// End of file
    Eof,
    /// Error
    Error,
}

impl TokenType for DsvTokenType {
    const END_OF_STREAM: Self = Self::Eof;
    type Role = UniversalTokenRole;

    fn is_ignored(&self) -> bool {
        matches!(self, Self::Whitespace)
    }

    fn is_comment(&self) -> bool {
        false
    }

    fn is_whitespace(&self) -> bool {
        matches!(self, Self::Whitespace | Self::Newline)
    }

    fn role(&self) -> Self::Role {
        match self {
            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
            Self::Field | Self::QuotedField | Self::UnquotedField => UniversalTokenRole::Literal,
            Self::Separator | Self::Quote | Self::EscapedQuote => UniversalTokenRole::Punctuation,
            Self::Eof => UniversalTokenRole::Eof,
            Self::Error => UniversalTokenRole::Error,
        }
    }
}