rtoon 0.1.1

Token-Oriented Object Notation - A compact, human-readable format for LLM data with 30-60% fewer tokens than JSON
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

pub mod parser;
pub mod scanner;
pub mod validation;

use serde_json::Value;

use crate::{
    error::ToonResult,
    types::DecodeOptions,
};

pub fn decode(input: &str, options: &DecodeOptions) -> ToonResult<Value> {
    let mut parser = parser::Parser::new(input, options.clone());
    parser.parse()
}

pub fn decode_strict(input: &str) -> ToonResult<Value> {
    decode(input, &DecodeOptions::new().with_strict(true))
}

pub fn decode_strict_with_options(input: &str, options: &DecodeOptions) -> ToonResult<Value> {
    let opts = options.clone().with_strict(true);
    decode(input, &opts)
}

pub fn decode_no_coerce(input: &str) -> ToonResult<Value> {
    decode(input, &DecodeOptions::new().with_coerce_types(false))
}

pub fn decode_no_coerce_with_options(input: &str, options: &DecodeOptions) -> ToonResult<Value> {
    let opts = options.clone().with_coerce_types(false);
    decode(input, &opts)
}

pub fn decode_default(input: &str) -> ToonResult<Value> {
    decode(input, &DecodeOptions::default())
}

#[cfg(test)]
mod tests {
    use serde_json::json;

    use super::*;

    #[test]
    fn test_decode_null() {
        assert_eq!(decode_default("null").unwrap(), json!(null));
    }

    #[test]
    fn test_decode_bool() {
        assert_eq!(decode_default("true").unwrap(), json!(true));
        assert_eq!(decode_default("false").unwrap(), json!(false));
    }

    #[test]
    fn test_decode_number() {
        assert_eq!(decode_default("42").unwrap(), json!(42));
        assert_eq!(decode_default("3.14").unwrap(), json!(3.14));
        assert_eq!(decode_default("-5").unwrap(), json!(-5));
    }

    #[test]
    fn test_decode_string() {
        assert_eq!(decode_default("hello").unwrap(), json!("hello"));
        assert_eq!(
            decode_default("\"hello world\"").unwrap(),
            json!("hello world")
        );
    }

    #[test]
    fn test_decode_simple_object() {
        let input = "name: Alice\nage: 30";
        let result = decode_default(input).unwrap();
        assert_eq!(result["name"], json!("Alice"));
        assert_eq!(result["age"], json!(30));
    }

    #[test]
    fn test_decode_primitive_array() {
        let input = "tags[3]: reading,gaming,coding";
        let result = decode_default(input).unwrap();
        assert_eq!(result["tags"], json!(["reading", "gaming", "coding"]));
    }

    #[test]
    fn test_decode_tabular_array() {
        let input = "users[2]{id,name,role}:\n  1,Alice,admin\n  2,Bob,user";
        let result = decode_default(input).unwrap();
        assert_eq!(
            result["users"],
            json!([
                {"id": 1, "name": "Alice", "role": "admin"},
                {"id": 2, "name": "Bob", "role": "user"}
            ])
        );
    }

    #[test]
    fn test_decode_empty_array() {
        let input = "items[0]:";
        let result = decode_default(input).unwrap();
        assert_eq!(result["items"], json!([]));
    }

    #[test]
    fn test_decode_quoted_strings() {
        let input = "tags[3]: \"true\",\"42\",\"-3.14\"";
        let result = decode_default(input).unwrap();
        assert_eq!(result["tags"], json!(["true", "42", "-3.14"]));
    }
}