rtoon 0.1.1

Token-Oriented Object Notation - A compact, human-readable format for LLM data with 30-60% fewer tokens than JSON
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use crate::error::{
    ToonError,
    ToonResult,
};

pub fn validate_array_length(expected: usize, actual: usize, strict: bool) -> ToonResult<()> {
    if strict && expected != actual {
        return Err(ToonError::length_mismatch(expected, actual));
    }
    Ok(())
}

pub fn validate_field_list(fields: &[String]) -> ToonResult<()> {
    if fields.is_empty() {
        return Err(ToonError::InvalidInput(
            "Field list cannot be empty for tabular arrays".to_string(),
        ));
    }

    for i in 0..fields.len() {
        for j in (i + 1)..fields.len() {
            if fields[i] == fields[j] {
                return Err(ToonError::InvalidInput(format!(
                    "Duplicate field name: '{}'",
                    fields[i]
                )));
            }
        }
    }

    for field in fields {
        if field.is_empty() {
            return Err(ToonError::InvalidInput(
                "Field name cannot be empty".to_string(),
            ));
        }
    }

    Ok(())
}

pub fn validate_row_length(
    row_index: usize,
    expected_fields: usize,
    actual_values: usize,
) -> ToonResult<()> {
    if expected_fields != actual_values {
        return Err(ToonError::InvalidStructure(format!(
            "Row {} has {} values but expected {} fields",
            row_index, actual_values, expected_fields
        )));
    }
    Ok(())
}

pub fn validate_delimiter_consistency(
    detected: Option<crate::types::Delimiter>,
    expected: Option<crate::types::Delimiter>,
) -> ToonResult<()> {
    if let (Some(detected), Some(expected)) = (detected, expected) {
        if detected != expected {
            return Err(ToonError::InvalidDelimiter(format!(
                "Detected delimiter {:?} but expected {:?}",
                detected, expected
            )));
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_validate_array_length() {
        assert!(validate_array_length(5, 3, false).is_ok());
        assert!(validate_array_length(3, 5, false).is_ok());

        assert!(validate_array_length(5, 5, true).is_ok());
        assert!(validate_array_length(5, 3, true).is_err());
        assert!(validate_array_length(3, 5, true).is_err());
    }

    #[test]
    fn test_validate_field_list() {
        assert!(validate_field_list(&["id".to_string(), "name".to_string()]).is_ok());
        assert!(validate_field_list(&["field1".to_string()]).is_ok());

        assert!(validate_field_list(&[]).is_err());

        assert!(
            validate_field_list(&["id".to_string(), "name".to_string(), "id".to_string()]).is_err()
        );

        assert!(
            validate_field_list(&["id".to_string(), "".to_string(), "name".to_string()]).is_err()
        );
    }

    #[test]
    fn test_validate_row_length() {
        assert!(validate_row_length(0, 3, 3).is_ok());
        assert!(validate_row_length(1, 5, 5).is_ok());

        assert!(validate_row_length(0, 3, 2).is_err());
        assert!(validate_row_length(1, 3, 4).is_err());
    }

    #[test]
    fn test_validate_delimiter_consistency() {
        use crate::types::Delimiter;

        assert!(
            validate_delimiter_consistency(Some(Delimiter::Comma), Some(Delimiter::Comma)).is_ok()
        );

        assert!(
            validate_delimiter_consistency(Some(Delimiter::Comma), Some(Delimiter::Pipe)).is_err()
        );

        assert!(validate_delimiter_consistency(None, Some(Delimiter::Comma)).is_ok());
        assert!(validate_delimiter_consistency(Some(Delimiter::Comma), None).is_ok());
        assert!(validate_delimiter_consistency(None, None).is_ok());
    }
}