toon_format/
lib.rs

1//! # TOON Format for Rust
2//!
3//! Token-Oriented Object Notation (TOON) is a compact, human-readable format
4//! designed for passing structured data to Large Language Models with
5//! significantly reduced token usage.
6//!
7//! This crate reserves the `toon-format` namespace for the official Rust
8//! implementation. Full implementation coming soon!
9//!
10//! ## Resources
11//!
12//! - [TOON Specification](https://github.com/johannschopplich/toon/blob/main/SPEC.md)
13//! - [Main Repository](https://github.com/johannschopplich/toon)
14//! - [Other Implementations](https://github.com/johannschopplich/toon#other-implementations)
15//!
16//! ## Example Usage (Future)
17//!
18//! ```ignore
19//! use toon_format::{encode, decode};
20//!
21//! let data = json!({"name": "Alice", "age": 30});
22//! let toon_string = encode(&data)?;
23//! let decoded = decode(&toon_string)?;
24//! # Ok::<(), toon_format::ToonError>(())
25//! ```
26#![warn(rustdoc::missing_crate_level_docs)]
27
28pub mod constants;
29pub mod decode;
30pub mod encode;
31#[cfg(feature = "cli")]
32pub mod tui;
33pub mod types;
34pub mod utils;
35
36pub use decode::{
37    decode,
38    decode_default,
39    decode_no_coerce,
40    decode_no_coerce_with_options,
41    decode_strict,
42    decode_strict_with_options,
43};
44#[cfg(feature = "json_stream")]
45pub use encode::json_stream::{
46    encode_json_reader,
47    encode_json_reader_default,
48    encode_json_stream,
49    encode_json_stream_default,
50    StreamingEncodeOptions,
51};
52pub use encode::{
53    encode,
54    encode_array,
55    encode_default,
56    encode_object,
57};
58pub use types::{
59    DecodeOptions,
60    Delimiter,
61    EncodeOptions,
62    Indent,
63    ToonError,
64};
65pub use utils::{
66    literal::{
67        is_keyword,
68        is_literal_like,
69    },
70    normalize,
71    string::{
72        escape_string,
73        is_valid_unquoted_key,
74        needs_quoting,
75    },
76};
77
78#[cfg(test)]
79mod tests {
80    use serde_json::{
81        json,
82        Value,
83    };
84
85    use crate::{
86        constants::is_keyword,
87        decode::{
88            decode_default,
89            decode_strict,
90        },
91        encode::{
92            encode,
93            encode_default,
94        },
95        types::{
96            Delimiter,
97            EncodeOptions,
98        },
99        utils::{
100            escape_string,
101            is_literal_like,
102            needs_quoting,
103            normalize,
104        },
105    };
106
107    #[test]
108    fn test_round_trip_simple() {
109        let original = json!({"name": "Alice", "age": 30});
110        let encoded = encode_default(&original).unwrap();
111        let decoded: Value = decode_default(&encoded).unwrap();
112        assert_eq!(original, decoded);
113    }
114
115    #[test]
116    fn test_round_trip_array() {
117        let original = json!({"tags": ["reading", "gaming", "coding"]});
118        let encoded = encode_default(&original).unwrap();
119        let decoded: Value = decode_default(&encoded).unwrap();
120        assert_eq!(original, decoded);
121    }
122
123    #[test]
124    fn test_round_trip_tabular() {
125        let original = json!({
126            "users": [
127                {"id": 1, "name": "Alice", "role": "admin"},
128                {"id": 2, "name": "Bob", "role": "user"}
129            ]
130        });
131        let encoded = encode_default(&original).unwrap();
132        let decoded: Value = decode_default(&encoded).unwrap();
133        assert_eq!(original, decoded);
134    }
135
136    #[test]
137    fn test_custom_delimiter() {
138        let original = json!({"tags": ["a", "b", "c"]});
139        let opts = EncodeOptions::new().with_delimiter(Delimiter::Pipe);
140        let encoded = encode(&original, &opts).unwrap();
141        assert!(encoded.contains("|"));
142
143        let decoded: Value = decode_default(&encoded).unwrap();
144        assert_eq!(original, decoded);
145    }
146
147    #[test]
148    fn test_decode_strict_helper() {
149        let input = "items[2]: a,b";
150        assert!(decode_strict::<Value>(input).is_ok());
151
152        let input = "items[3]: a,b";
153        assert!(decode_strict::<Value>(input).is_err());
154    }
155
156    #[test]
157    fn test_normalize_exported() {
158        let value = json!(f64::NAN);
159        let normalized = normalize(value.into());
160        assert_eq!(serde_json::Value::from(normalized), json!(null));
161    }
162
163    #[test]
164    fn test_utilities_exported() {
165        assert!(is_keyword("null"));
166        assert!(is_literal_like("true"));
167        assert_eq!(escape_string("hello\nworld"), "hello\\nworld");
168        assert!(needs_quoting("true", Delimiter::Comma.as_char()));
169    }
170
171    use serde::{
172        Deserialize,
173        Serialize,
174    };
175
176    #[derive(Debug, Serialize, Deserialize, PartialEq)]
177    struct TestUser {
178        name: String,
179        age: u32,
180        active: bool,
181    }
182
183    #[test]
184    fn test_encode_decode_simple_struct() {
185        use crate::{
186            decode_default,
187            encode_default,
188        };
189
190        let user = TestUser {
191            name: "Alice".to_string(),
192            age: 30,
193            active: true,
194        };
195
196        let toon = encode_default(&user).unwrap();
197        assert!(toon.contains("name: Alice"));
198        assert!(toon.contains("age: 30"));
199        assert!(toon.contains("active: true"));
200
201        let decoded: TestUser = decode_default(&toon).unwrap();
202        assert_eq!(user, decoded);
203    }
204
205    #[derive(Debug, Serialize, Deserialize, PartialEq)]
206    struct TestProduct {
207        id: u64,
208        name: String,
209        tags: Vec<String>,
210    }
211
212    #[test]
213    fn test_encode_decode_with_array() {
214        use crate::{
215            decode_default,
216            encode_default,
217        };
218
219        let product = TestProduct {
220            id: 42,
221            name: "Widget".to_string(),
222            tags: vec!["electronics".to_string(), "gadgets".to_string()],
223        };
224
225        let toon = encode_default(&product).unwrap();
226        let decoded: TestProduct = decode_default(&toon).unwrap();
227        assert_eq!(product, decoded);
228    }
229
230    #[test]
231    fn test_encode_decode_vec_of_structs() {
232        use crate::{
233            decode_default,
234            encode_default,
235        };
236
237        let users = vec![
238            TestUser {
239                name: "Alice".to_string(),
240                age: 30,
241                active: true,
242            },
243            TestUser {
244                name: "Bob".to_string(),
245                age: 25,
246                active: false,
247            },
248        ];
249
250        let toon = encode_default(&users).unwrap();
251        let decoded: Vec<TestUser> = decode_default(&toon).unwrap();
252        assert_eq!(users, decoded);
253    }
254
255    #[derive(Debug, Serialize, Deserialize, PartialEq)]
256    struct Nested {
257        outer: OuterStruct,
258    }
259
260    #[derive(Debug, Serialize, Deserialize, PartialEq)]
261    struct OuterStruct {
262        inner: InnerStruct,
263        value: i32,
264    }
265
266    #[derive(Debug, Serialize, Deserialize, PartialEq)]
267    struct InnerStruct {
268        data: String,
269    }
270
271    #[test]
272    fn test_encode_decode_nested_structs() {
273        use crate::{
274            decode_default,
275            encode_default,
276        };
277
278        let nested = Nested {
279            outer: OuterStruct {
280                inner: InnerStruct {
281                    data: "test".to_string(),
282                },
283                value: 42,
284            },
285        };
286
287        let toon = encode_default(&nested).unwrap();
288        let decoded: Nested = decode_default(&toon).unwrap();
289        assert_eq!(nested, decoded);
290    }
291
292    #[test]
293    fn test_round_trip_list_item_tabular_v3() {
294        use crate::{
295            decode_default,
296            encode_default,
297        };
298
299        let original = json!({
300            "items": [
301                {
302                    "users": [
303                        {"id": 1, "name": "Alice", "role": "admin"},
304                        {"id": 2, "name": "Bob", "role": "user"}
305                    ],
306                    "status": "active",
307                    "count": 2
308                }
309            ]
310        });
311
312        let encoded = encode_default(&original).unwrap();
313        let decoded: Value = decode_default(&encoded).unwrap();
314
315        assert_eq!(original, decoded);
316    }
317
318    #[test]
319    fn test_round_trip_complex_list_item_tabular_v3() {
320        use crate::{
321            decode_default,
322            encode_default,
323        };
324
325        let original = json!({
326            "data": [
327                {
328                    "records": [
329                        {"id": 1, "value": "x", "score": 100},
330                        {"id": 2, "value": "y", "score": 200}
331                    ],
332                    "total": 2,
333                    "status": "active"
334                },
335                {
336                    "records": [
337                        {"id": 3, "value": "z", "score": 300}
338                    ],
339                    "total": 1,
340                    "status": "pending"
341                }
342            ]
343        });
344
345        let encoded = encode_default(&original).unwrap();
346        let decoded: Value = decode_default(&encoded).unwrap();
347
348        assert_eq!(original, decoded);
349    }
350
351    #[test]
352    fn test_round_trip_mixed_list_items_v3() {
353        use crate::{
354            decode_default,
355            encode_default,
356        };
357
358        let original = json!({
359            "entries": [
360                {
361                    "type": "simple",
362                    "value": 42
363                },
364                {
365                    "people": [
366                        {"name": "Alice", "age": 30},
367                        {"name": "Bob", "age": 25}
368                    ],
369                    "type": "complex"
370                },
371                {
372                    "tags": ["a", "b", "c"],
373                    "type": "array"
374                }
375            ]
376        });
377
378        let encoded = encode_default(&original).unwrap();
379        let decoded: Value = decode_default(&encoded).unwrap();
380
381        assert_eq!(original, decoded);
382    }
383}
toon_format/lib.rs

toon_format/
lib.rs
