base_d/encoders/algorithms/schema/
mod.rs

1pub mod binary_packer;
2pub mod binary_unpacker;
3pub mod compression;
4pub mod display96;
5pub mod fiche;
6pub mod frame;
7pub mod parsers;
8pub mod serializers;
9pub mod types;
10
11#[cfg(test)]
12mod edge_cases;
13
14// Re-export key types for convenience
15pub use binary_packer::pack;
16pub use binary_unpacker::unpack;
17pub use compression::SchemaCompressionAlgo;
18pub use frame::{decode_framed, encode_framed};
19pub use parsers::{InputParser, JsonParser};
20pub use serializers::{JsonSerializer, OutputSerializer};
21pub use types::{
22    FieldDef, FieldType, IntermediateRepresentation, SchemaError, SchemaHeader, SchemaValue,
23};
24
25// Re-export fiche functions for library users
26#[allow(unused_imports)]
27pub use fiche::{parse as parse_fiche, serialize as serialize_fiche};
28
29/// Encode JSON to schema format: JSON → IR → binary → \[compress\] → display96 → framed
30///
31/// Transforms JSON into a compact, display-safe wire format suitable for LLM-to-LLM communication.
32/// The output is wrapped in Egyptian hieroglyph delimiters (`𓍹...𓍺`) and uses a 96-character
33/// alphabet of box-drawing and geometric shapes.
34///
35/// # Arguments
36///
37/// * `json` - JSON string to encode (must be object or array of objects)
38/// * `compress` - Optional compression algorithm (brotli, lz4, or zstd)
39///
40/// # Returns
41///
42/// Returns a framed, display-safe string like `𓍹{encoded_payload}𓍺`
43///
44/// # Errors
45///
46/// * `SchemaError::InvalidInput` - Invalid JSON or unsupported structure (e.g., root primitives)
47/// * `SchemaError::Compression` - Compression failure
48///
49/// # Example
50///
51/// ```ignore
52/// use base_d::{encode_schema, SchemaCompressionAlgo};
53///
54/// let json = r#"{"users":[{"id":1,"name":"alice"}]}"#;
55///
56/// // Without compression
57/// let encoded = encode_schema(json, None)?;
58/// println!("{}", encoded); // 𓍹╣◟╥◕◝▰◣◥▟╺▖◘▰◝▤◀╧𓍺
59///
60/// // With brotli compression
61/// let compressed = encode_schema(json, Some(SchemaCompressionAlgo::Brotli))?;
62/// ```
63///
64/// # See Also
65///
66/// * [`decode_schema`] - Decode schema format back to JSON
67/// * [SCHEMA.md](../../../SCHEMA.md) - Full format specification
68pub fn encode_schema(
69    json: &str,
70    compress: Option<SchemaCompressionAlgo>,
71) -> Result<String, SchemaError> {
72    use parsers::{InputParser, JsonParser};
73
74    let ir = JsonParser::parse(json)?;
75    let binary = pack(&ir);
76    let compressed = compression::compress_with_prefix(&binary, compress)?;
77    Ok(frame::encode_framed(&compressed))
78}
79
80/// Decode schema format to JSON: framed → display96 → \[decompress\] → binary → IR → JSON
81///
82/// Reverses the schema encoding pipeline to reconstruct the original JSON from the framed,
83/// display-safe wire format. Automatically detects and handles compression.
84///
85/// # Arguments
86///
87/// * `encoded` - Schema-encoded string with delimiters (`𓍹...𓍺`)
88/// * `pretty` - Pretty-print JSON output with indentation
89///
90/// # Returns
91///
92/// Returns the decoded JSON string (minified or pretty-printed)
93///
94/// # Errors
95///
96/// * `SchemaError::InvalidFrame` - Missing or invalid frame delimiters
97/// * `SchemaError::InvalidCharacter` - Invalid character in display96 payload
98/// * `SchemaError::Decompression` - Decompression failure
99/// * `SchemaError::UnexpectedEndOfData` - Truncated or corrupted binary data
100/// * `SchemaError::InvalidTypeTag` - Invalid type tag in header
101///
102/// # Example
103///
104/// ```ignore
105/// use base_d::decode_schema;
106///
107/// let encoded = "𓍹╣◟╥◕◝▰◣◥▟╺▖◘▰◝▤◀╧𓍺";
108///
109/// // Minified output
110/// let json = decode_schema(encoded, false)?;
111/// println!("{}", json); // {"users":[{"id":1,"name":"alice"}]}
112///
113/// // Pretty-printed output
114/// let pretty = decode_schema(encoded, true)?;
115/// println!("{}", pretty);
116/// // {
117/// //   "users": [
118/// //     {"id": 1, "name": "alice"}
119/// //   ]
120/// // }
121/// ```
122///
123/// # See Also
124///
125/// * [`encode_schema`] - Encode JSON to schema format
126/// * [SCHEMA.md](../../../SCHEMA.md) - Full format specification
127pub fn decode_schema(encoded: &str, pretty: bool) -> Result<String, SchemaError> {
128    use serializers::{JsonSerializer, OutputSerializer};
129
130    let compressed = frame::decode_framed(encoded)?;
131    let binary = compression::decompress_with_prefix(&compressed)?;
132    let ir = unpack(&binary)?;
133    JsonSerializer::serialize(&ir, pretty)
134}
135
136/// Encode JSON to fiche format: JSON → IR → fiche
137///
138/// Transforms JSON into a model-readable structured format using Unicode delimiters.
139/// Unlike carrier98 (opaque binary), fiche is designed for models to parse directly.
140///
141/// # Format
142///
143/// ```text
144/// @{root}┃{field}:{type}┃{field}:{type}...
145/// ◉{value}┃{value}┃{value}...
146/// ```
147///
148/// # Example
149///
150/// ```ignore
151/// use base_d::encode_fiche;
152///
153/// let json = r#"{"users":[{"id":1,"name":"alice"}]}"#;
154/// let fiche = encode_fiche(json)?;
155/// // @users┃id:int┃name:str
156/// // ◉1┃alice
157/// ```
158pub fn encode_fiche(json: &str) -> Result<String, SchemaError> {
159    encode_fiche_with_options(json, false)
160}
161
162pub fn encode_fiche_minified(json: &str) -> Result<String, SchemaError> {
163    encode_fiche_with_options(json, true)
164}
165
166fn encode_fiche_with_options(json: &str, minify: bool) -> Result<String, SchemaError> {
167    use parsers::{InputParser, JsonParser};
168
169    let ir = JsonParser::parse(json)?;
170    if minify {
171        fiche::serialize_minified(&ir)
172    } else {
173        fiche::serialize(&ir)
174    }
175}
176
177/// Decode fiche format to JSON: fiche → IR → JSON
178///
179/// Reverses the fiche encoding to reconstruct JSON from the model-readable format.
180///
181/// # Example
182///
183/// ```ignore
184/// use base_d::decode_fiche;
185///
186/// let fiche = "@users┃id:int┃name:str\n◉1┃alice";
187/// let json = decode_fiche(fiche, false)?;
188/// // {"users":[{"id":1,"name":"alice"}]}
189/// ```
190pub fn decode_fiche(fiche_input: &str, pretty: bool) -> Result<String, SchemaError> {
191    use serializers::{JsonSerializer, OutputSerializer};
192
193    let ir = fiche::parse(fiche_input)?;
194    JsonSerializer::serialize(&ir, pretty)
195}
196
197#[cfg(test)]
198mod integration_tests {
199    use super::*;
200    use crate::encoders::algorithms::schema::types::{
201        FLAG_HAS_NULLS, FLAG_HAS_ROOT_KEY, FieldDef, FieldType, IntermediateRepresentation,
202        SchemaHeader, SchemaValue,
203    };
204    use parsers::{InputParser, JsonParser};
205    use serializers::{JsonSerializer, OutputSerializer};
206
207    #[test]
208    fn test_round_trip_simple() {
209        let fields = vec![
210            FieldDef::new("id", FieldType::U64),
211            FieldDef::new("name", FieldType::String),
212        ];
213        let header = SchemaHeader::new(2, fields);
214
215        let values = vec![
216            SchemaValue::U64(1),
217            SchemaValue::String("Alice".to_string()),
218            SchemaValue::U64(2),
219            SchemaValue::String("Bob".to_string()),
220        ];
221
222        let original = IntermediateRepresentation::new(header, values).unwrap();
223
224        // Pack and unpack
225        let packed = pack(&original);
226        let unpacked = unpack(&packed).unwrap();
227
228        assert_eq!(original, unpacked);
229    }
230
231    #[test]
232    fn test_round_trip_all_types() {
233        let fields = vec![
234            FieldDef::new("u64_field", FieldType::U64),
235            FieldDef::new("i64_field", FieldType::I64),
236            FieldDef::new("f64_field", FieldType::F64),
237            FieldDef::new("string_field", FieldType::String),
238            FieldDef::new("bool_field", FieldType::Bool),
239        ];
240        let header = SchemaHeader::new(1, fields);
241
242        let values = vec![
243            SchemaValue::U64(42),
244            SchemaValue::I64(-42),
245            SchemaValue::F64(std::f64::consts::PI),
246            SchemaValue::String("test".to_string()),
247            SchemaValue::Bool(true),
248        ];
249
250        let original = IntermediateRepresentation::new(header, values).unwrap();
251
252        let packed = pack(&original);
253        let unpacked = unpack(&packed).unwrap();
254
255        assert_eq!(original, unpacked);
256    }
257
258    #[test]
259    fn test_round_trip_with_root_key() {
260        let mut header = SchemaHeader::new(1, vec![FieldDef::new("id", FieldType::U64)]);
261        header.root_key = Some("users".to_string());
262        header.set_flag(FLAG_HAS_ROOT_KEY);
263
264        let values = vec![SchemaValue::U64(42)];
265        let original = IntermediateRepresentation::new(header, values).unwrap();
266
267        let packed = pack(&original);
268        let unpacked = unpack(&packed).unwrap();
269
270        assert_eq!(original, unpacked);
271    }
272
273    #[test]
274    fn test_round_trip_with_nulls() {
275        let mut header = SchemaHeader::new(
276            2,
277            vec![
278                FieldDef::new("id", FieldType::U64),
279                FieldDef::new("name", FieldType::String),
280            ],
281        );
282
283        // Mark second value as null (row 0, field 1)
284        let total_values: usize = 2 * 2; // 2 rows * 2 fields = 4 values
285        let bitmap_bytes = total_values.div_ceil(8); // 1 byte
286        let mut null_bitmap = vec![0u8; bitmap_bytes];
287        null_bitmap[0] |= 1 << 1; // Set bit 1 (second value)
288
289        header.null_bitmap = Some(null_bitmap);
290        header.set_flag(FLAG_HAS_NULLS);
291
292        let values = vec![
293            SchemaValue::U64(1),
294            SchemaValue::Null, // This is marked as null in bitmap
295            SchemaValue::U64(2),
296            SchemaValue::String("Bob".to_string()),
297        ];
298
299        let original = IntermediateRepresentation::new(header, values).unwrap();
300
301        let packed = pack(&original);
302        let unpacked = unpack(&packed).unwrap();
303
304        assert_eq!(original, unpacked);
305    }
306
307    #[test]
308    fn test_round_trip_array() {
309        let fields = vec![FieldDef::new(
310            "tags",
311            FieldType::Array(Box::new(FieldType::U64)),
312        )];
313        let header = SchemaHeader::new(1, fields);
314
315        let values = vec![SchemaValue::Array(vec![
316            SchemaValue::U64(1),
317            SchemaValue::U64(2),
318            SchemaValue::U64(3),
319        ])];
320
321        let original = IntermediateRepresentation::new(header, values).unwrap();
322
323        let packed = pack(&original);
324        let unpacked = unpack(&packed).unwrap();
325
326        assert_eq!(original, unpacked);
327    }
328
329    #[test]
330    fn test_round_trip_large_values() {
331        let fields = vec![
332            FieldDef::new("large_u64", FieldType::U64),
333            FieldDef::new("large_i64", FieldType::I64),
334        ];
335        let header = SchemaHeader::new(1, fields);
336
337        let values = vec![SchemaValue::U64(u64::MAX), SchemaValue::I64(i64::MIN)];
338
339        let original = IntermediateRepresentation::new(header, values).unwrap();
340
341        let packed = pack(&original);
342        let unpacked = unpack(&packed).unwrap();
343
344        assert_eq!(original, unpacked);
345    }
346
347    #[test]
348    fn test_round_trip_empty_string() {
349        let fields = vec![FieldDef::new("name", FieldType::String)];
350        let header = SchemaHeader::new(1, fields);
351
352        let values = vec![SchemaValue::String("".to_string())];
353
354        let original = IntermediateRepresentation::new(header, values).unwrap();
355
356        let packed = pack(&original);
357        let unpacked = unpack(&packed).unwrap();
358
359        assert_eq!(original, unpacked);
360    }
361
362    #[test]
363    fn test_round_trip_multiple_rows() {
364        let fields = vec![
365            FieldDef::new("id", FieldType::U64),
366            FieldDef::new("score", FieldType::F64),
367            FieldDef::new("active", FieldType::Bool),
368        ];
369        let header = SchemaHeader::new(3, fields);
370
371        let values = vec![
372            SchemaValue::U64(1),
373            SchemaValue::F64(95.5),
374            SchemaValue::Bool(true),
375            SchemaValue::U64(2),
376            SchemaValue::F64(87.3),
377            SchemaValue::Bool(false),
378            SchemaValue::U64(3),
379            SchemaValue::F64(92.1),
380            SchemaValue::Bool(true),
381        ];
382
383        let original = IntermediateRepresentation::new(header, values).unwrap();
384
385        let packed = pack(&original);
386        let unpacked = unpack(&packed).unwrap();
387
388        assert_eq!(original, unpacked);
389    }
390
391    #[test]
392    fn test_invalid_data() {
393        // Empty data
394        let result = unpack(&[]);
395        assert!(matches!(
396            result,
397            Err(SchemaError::UnexpectedEndOfData { .. })
398        ));
399
400        // Truncated data
401        let result = unpack(&[0, 1, 2]);
402        assert!(result.is_err());
403    }
404
405    #[test]
406    fn test_json_full_roundtrip() {
407        let input = r#"{"users":[{"id":1,"name":"alice"},{"id":2,"name":"bob"}]}"#;
408        let ir = JsonParser::parse(input).unwrap();
409        let binary = pack(&ir);
410        let compressed = compression::compress_with_prefix(&binary, None).unwrap();
411        let decompressed = compression::decompress_with_prefix(&compressed).unwrap();
412        let ir2 = unpack(&decompressed).unwrap();
413        let output = JsonSerializer::serialize(&ir2, false).unwrap();
414
415        // Parse both as serde_json::Value and compare (order-independent)
416        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
417        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
418        assert_eq!(input_value, output_value);
419    }
420
421    #[test]
422    fn test_json_simple_object() {
423        let input = r#"{"id":1,"name":"alice","score":95.5}"#;
424        let ir = JsonParser::parse(input).unwrap();
425        let binary = pack(&ir);
426        let ir2 = unpack(&binary).unwrap();
427        let output = JsonSerializer::serialize(&ir2, false).unwrap();
428
429        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
430        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
431        assert_eq!(input_value, output_value);
432    }
433
434    #[test]
435    fn test_json_swapi_nested_arrays() {
436        // SWAPI-like data with nested arrays of primitives
437        // Arrays now flatten to indexed fields
438        let input = r#"{"people":[{"name":"Luke","height":"172","films":["film/1","film/2"],"vehicles":[]},{"name":"C-3PO","height":"167","films":["film/1","film/2","film/3"],"vehicles":[]}]}"#;
439        let ir = JsonParser::parse(input).unwrap();
440
441        // Verify fiche representation
442        let fiche_output = fiche::serialize(&ir).unwrap();
443
444        // Should have @people root key
445        assert!(fiche_output.starts_with("@people"));
446        // Arrays now flatten, so films.0, films.1, etc. (no @ marker)
447        assert!(fiche_output.contains("films჻0:str") || fiche_output.contains("films.0:str"));
448        // vehicles is empty array, so no fields generated
449
450        // Verify round trip - arrays become indexed objects
451        let binary = pack(&ir);
452        let ir2 = unpack(&binary).unwrap();
453        let output = JsonSerializer::serialize(&ir2, false).unwrap();
454
455        // Parse output and verify structure
456        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
457        let people = output_value
458            .as_object()
459            .unwrap()
460            .get("people")
461            .unwrap()
462            .as_array()
463            .unwrap();
464
465        // First person has films as properly reconstructed array
466        let luke = &people[0];
467        assert_eq!(luke["name"], "Luke");
468        assert_eq!(luke["height"], "172");
469        let luke_films = luke["films"].as_array().unwrap();
470        assert_eq!(luke_films[0], "film/1");
471        assert_eq!(luke_films[1], "film/2");
472    }
473
474    #[test]
475    fn test_json_wrapper_keys() {
476        // Test common pagination wrapper keys get unwrapped
477        let test_cases = vec![
478            r#"{"results":[{"id":1,"name":"a"},{"id":2,"name":"b"}]}"#,
479            r#"{"data":[{"id":1,"name":"a"},{"id":2,"name":"b"}]}"#,
480            r#"{"items":[{"id":1,"name":"a"},{"id":2,"name":"b"}]}"#,
481            r#"{"records":[{"id":1,"name":"a"},{"id":2,"name":"b"}]}"#,
482        ];
483
484        for input in test_cases {
485            let ir = JsonParser::parse(input).unwrap();
486
487            // Should have root key from wrapper
488            assert!(ir.header.root_key.is_some());
489            let root = ir.header.root_key.as_ref().unwrap();
490            assert!(root == "results" || root == "data" || root == "items" || root == "records");
491
492            // Should have 2 rows (unwrapped the array)
493            assert_eq!(ir.header.row_count, 2);
494
495            // Round trip should preserve data
496            let binary = pack(&ir);
497            let ir2 = unpack(&binary).unwrap();
498            let output = JsonSerializer::serialize(&ir2, false).unwrap();
499
500            let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
501            let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
502            assert_eq!(input_value, output_value);
503        }
504    }
505
506    #[test]
507    fn test_json_nested_objects() {
508        let input = r#"{"user":{"profile":{"name":"alice","age":30}}}"#;
509        let ir = JsonParser::parse(input).unwrap();
510        let binary = pack(&ir);
511        let ir2 = unpack(&binary).unwrap();
512        let output = JsonSerializer::serialize(&ir2, false).unwrap();
513
514        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
515        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
516        assert_eq!(input_value, output_value);
517    }
518
519    #[test]
520    fn test_json_with_nulls() {
521        let input = r#"{"name":"alice","age":null,"active":true}"#;
522        let ir = JsonParser::parse(input).unwrap();
523        assert!(ir.header.has_flag(FLAG_HAS_NULLS));
524
525        let binary = pack(&ir);
526        let ir2 = unpack(&binary).unwrap();
527        let output = JsonSerializer::serialize(&ir2, false).unwrap();
528
529        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
530        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
531        assert_eq!(input_value, output_value);
532    }
533
534    #[test]
535    fn test_json_with_arrays() {
536        // Arrays now flatten to indexed objects
537        let input = r#"{"scores":[95,87,92],"tags":["rust","json"]}"#;
538        let ir = JsonParser::parse(input).unwrap();
539        let binary = pack(&ir);
540        let ir2 = unpack(&binary).unwrap();
541        let output = JsonSerializer::serialize(&ir2, false).unwrap();
542
543        // Expected: arrays are properly reconstructed as arrays
544        let expected = r#"{"scores":[95,87,92],"tags":["rust","json"]}"#;
545        let expected_value: serde_json::Value = serde_json::from_str(expected).unwrap();
546        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
547        assert_eq!(expected_value, output_value);
548    }
549
550    #[test]
551    fn test_encode_schema_roundtrip() {
552        let input = r#"{"users":[{"id":1,"name":"alice"},{"id":2,"name":"bob"}]}"#;
553        let encoded = encode_schema(input, None).unwrap();
554
555        // Validate frame delimiters
556        assert!(encoded.starts_with(frame::FRAME_START));
557        assert!(encoded.ends_with(frame::FRAME_END));
558
559        // Decode back to JSON
560        let decoded = decode_schema(&encoded, false).unwrap();
561
562        // Compare as JSON values (order-independent)
563        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
564        let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
565        assert_eq!(input_value, output_value);
566    }
567
568    #[test]
569    fn test_encode_schema_simple() {
570        let input = r#"{"id":1,"name":"alice","score":95.5}"#;
571        let encoded = encode_schema(input, None).unwrap();
572        let decoded = decode_schema(&encoded, false).unwrap();
573
574        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
575        let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
576        assert_eq!(input_value, output_value);
577    }
578
579    #[test]
580    fn test_encode_schema_with_nulls() {
581        let input = r#"{"name":"alice","age":null,"active":true}"#;
582        let encoded = encode_schema(input, None).unwrap();
583        let decoded = decode_schema(&encoded, false).unwrap();
584
585        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
586        let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
587        assert_eq!(input_value, output_value);
588    }
589
590    #[test]
591    fn test_encode_schema_empty_object() {
592        let input = r#"{}"#;
593        let result = encode_schema(input, None);
594        // Empty objects should fail or handle gracefully
595        // This depends on JsonParser behavior
596        println!("Empty object result: {:?}", result);
597    }
598
599    #[test]
600    fn test_decode_schema_invalid_frame() {
601        let invalid = "not_framed_data";
602        let result = decode_schema(invalid, false);
603        assert!(matches!(result, Err(SchemaError::InvalidFrame(_))));
604    }
605
606    #[test]
607    fn test_decode_schema_invalid_chars() {
608        let invalid = format!("{}ABC{}", frame::FRAME_START, frame::FRAME_END);
609        let result = decode_schema(&invalid, false);
610        assert!(matches!(result, Err(SchemaError::InvalidCharacter(_))));
611    }
612
613    #[test]
614    fn test_visual_wire_format() {
615        let input = r#"{"users":[{"id":1,"name":"alice"},{"id":2,"name":"bob"}]}"#;
616        let encoded = encode_schema(input, None).unwrap();
617
618        println!("\n=== Visual Wire Format ===");
619        println!("Input JSON: {}", input);
620        println!("Input length: {} bytes", input.len());
621        println!("\nEncoded output: {}", encoded);
622        println!(
623            "Encoded length: {} chars ({} bytes UTF-8)",
624            encoded.chars().count(),
625            encoded.len()
626        );
627
628        // Calculate compression ratio
629        let compression_ratio = input.len() as f64 / encoded.len() as f64;
630        println!("Compression ratio: {:.2}x", compression_ratio);
631
632        // Decode and verify
633        let decoded = decode_schema(&encoded, false).unwrap();
634        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
635        let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
636        assert_eq!(input_value, output_value);
637        println!("Roundtrip verified ✓\n");
638    }
639
640    #[test]
641    fn test_compression_comparison() {
642        let test_cases = [
643            r#"{"id":1}"#,
644            r#"{"id":1,"name":"alice"}"#,
645            r#"{"users":[{"id":1,"name":"alice"},{"id":2,"name":"bob"}]}"#,
646            r#"{"data":[1,2,3,4,5,6,7,8,9,10]}"#,
647        ];
648
649        println!("\n=== Compression Comparison ===");
650        for (i, input) in test_cases.iter().enumerate() {
651            let encoded = encode_schema(input, None).unwrap();
652            let ratio = input.len() as f64 / encoded.len() as f64;
653
654            println!(
655                "Test case {}: {} bytes → {} bytes ({:.2}x)",
656                i + 1,
657                input.len(),
658                encoded.len(),
659                ratio
660            );
661        }
662        println!();
663    }
664
665    #[test]
666    fn test_encode_schema_with_compression() {
667        use super::SchemaCompressionAlgo;
668
669        let input = r#"{"users":[{"id":1,"name":"alice"},{"id":2,"name":"bob"},{"id":3,"name":"charlie"}]}"#;
670
671        // Test each compression algorithm
672        for algo in [
673            SchemaCompressionAlgo::Brotli,
674            SchemaCompressionAlgo::Lz4,
675            SchemaCompressionAlgo::Zstd,
676        ] {
677            let encoded = encode_schema(input, Some(algo)).unwrap();
678            let decoded = decode_schema(&encoded, false).unwrap();
679
680            let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
681            let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
682            assert_eq!(
683                input_value, output_value,
684                "Failed for compression algorithm: {:?}",
685                algo
686            );
687        }
688    }
689
690    #[test]
691    fn test_compression_size_comparison() {
692        use super::SchemaCompressionAlgo;
693
694        let input = r#"{"users":[{"id":1,"name":"alice","active":true,"score":95.5},{"id":2,"name":"bob","active":false,"score":87.3},{"id":3,"name":"charlie","active":true,"score":92.1}]}"#;
695
696        println!("\n=== Compression Size Comparison ===");
697        println!("Input JSON: {} bytes", input.len());
698
699        let no_compress = encode_schema(input, None).unwrap();
700        println!("No compression: {} bytes", no_compress.len());
701
702        for algo in [
703            SchemaCompressionAlgo::Brotli,
704            SchemaCompressionAlgo::Lz4,
705            SchemaCompressionAlgo::Zstd,
706        ] {
707            let compressed = encode_schema(input, Some(algo)).unwrap();
708            let ratio = no_compress.len() as f64 / compressed.len() as f64;
709            println!(
710                "{:?}: {} bytes ({:.2}x vs uncompressed)",
711                algo,
712                compressed.len(),
713                ratio
714            );
715        }
716        println!();
717    }
718
719    #[test]
720    fn test_nested_object_roundtrip_single_level() {
721        let input = r#"{"id":"A1","name":"Jim","grade":{"math":60,"physics":66,"chemistry":61}}"#;
722
723        // JSON → IR → fiche
724        let ir = JsonParser::parse(input).unwrap();
725        let fiche = fiche::serialize(&ir).unwrap();
726
727        // Verify flattened field names with ჻
728        assert!(fiche.contains("grade჻math:int"));
729        assert!(fiche.contains("grade჻physics:int"));
730        assert!(fiche.contains("grade჻chemistry:int"));
731
732        // fiche → IR → JSON
733        let ir2 = fiche::parse(&fiche).unwrap();
734        let output = JsonSerializer::serialize(&ir2, false).unwrap();
735
736        // Compare JSON
737        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
738        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
739        assert_eq!(input_value, output_value);
740    }
741
742    #[test]
743    fn test_nested_object_roundtrip_deep() {
744        let input = r#"{"a":{"b":{"c":{"d":42}}}}"#;
745
746        let ir = JsonParser::parse(input).unwrap();
747        let fiche = fiche::serialize(&ir).unwrap();
748
749        // Verify deep nesting with ჻
750        assert!(fiche.contains("a჻b჻c჻d:int"));
751
752        let ir2 = fiche::parse(&fiche).unwrap();
753        let output = JsonSerializer::serialize(&ir2, false).unwrap();
754
755        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
756        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
757        assert_eq!(input_value, output_value);
758    }
759
760    #[test]
761    fn test_nested_object_roundtrip_array_of_objects() {
762        let input = r#"{"students":[{"id":"A1","name":"Jim","grade":{"math":60,"physics":66}},{"id":"B2","name":"Sara","grade":{"math":85,"physics":90}}]}"#;
763
764        let ir = JsonParser::parse(input).unwrap();
765        let fiche = fiche::serialize(&ir).unwrap();
766
767        // Verify root key and flattened nested fields
768        assert!(fiche.starts_with("@students"));
769        assert!(fiche.contains("grade჻math:int"));
770        assert!(fiche.contains("grade჻physics:int"));
771
772        let ir2 = fiche::parse(&fiche).unwrap();
773        let output = JsonSerializer::serialize(&ir2, false).unwrap();
774
775        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
776        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
777        assert_eq!(input_value, output_value);
778    }
779
780    #[test]
781    fn test_nested_object_roundtrip_mixed_with_arrays() {
782        // Arrays now flatten to indexed fields
783        let input = r#"{"person":{"name":"Alice","tags":["admin","user"],"address":{"city":"Boston","zip":"02101"}}}"#;
784
785        let ir = JsonParser::parse(input).unwrap();
786        let fiche = fiche::serialize(&ir).unwrap();
787
788        // Verify both object nesting and array flattening
789        assert!(fiche.contains("person჻name:str"));
790        // Arrays flatten, so tags.0, tags.1 (no @ marker)
791        assert!(fiche.contains("person჻tags჻0:str"));
792        assert!(fiche.contains("person჻address჻city:str"));
793        assert!(fiche.contains("person჻address჻zip:str"));
794
795        let ir2 = fiche::parse(&fiche).unwrap();
796        let output = JsonSerializer::serialize(&ir2, false).unwrap();
797
798        // Arrays are properly reconstructed
799        let expected = r#"{"person":{"address":{"city":"Boston","zip":"02101"},"name":"Alice","tags":["admin","user"]}}"#;
800        let expected_value: serde_json::Value = serde_json::from_str(expected).unwrap();
801        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
802        assert_eq!(expected_value, output_value);
803    }
804
805    #[test]
806    fn test_nested_object_roundtrip_schema_encode() {
807        let input = r#"{"data":{"user":{"profile":{"name":"alice","age":30}}}}"#;
808
809        // Full schema pipeline: JSON → IR → binary → display96 → framed
810        let encoded = encode_schema(input, None).unwrap();
811        let decoded = decode_schema(&encoded, false).unwrap();
812
813        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
814        let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
815        assert_eq!(input_value, output_value);
816    }
817}
base_d/encoders/algorithms/schema/mod.rs

base_d/encoders/algorithms/schema/
mod.rs
