base_d/encoders/algorithms/schema/
mod.rs

1pub mod binary_packer;
2pub mod binary_unpacker;
3pub mod display96;
4pub mod frame;
5pub mod parsers;
6pub mod serializers;
7pub mod types;
8
9// Re-export key types for convenience
10pub use binary_packer::pack;
11pub use binary_unpacker::unpack;
12pub use types::SchemaError;
13
14/// Full schema encoding pipeline: JSON → IR → binary → display96 → framed
15///
16/// # Example
17/// ```ignore
18/// let json = r#"{"users":[{"id":1,"name":"alice"}]}"#;
19/// let encoded = encode_schema(json)?;
20/// // Returns: "𓍹{display96_encoded}𓍺"
21/// ```
22pub fn encode_schema(json: &str) -> Result<String, SchemaError> {
23    use parsers::{InputParser, JsonParser};
24
25    let ir = JsonParser::parse(json)?;
26    let binary = pack(&ir);
27    Ok(frame::encode_framed(&binary))
28}
29
30/// Full schema decoding pipeline: framed → display96 → binary → IR → JSON
31///
32/// # Example
33/// ```ignore
34/// let framed = "𓍹{display96_encoded}𓍺";
35/// let json = decode_schema(framed)?;
36/// ```
37pub fn decode_schema(encoded: &str) -> Result<String, SchemaError> {
38    use serializers::{JsonSerializer, OutputSerializer};
39
40    let binary = frame::decode_framed(encoded)?;
41    let ir = unpack(&binary)?;
42    JsonSerializer::serialize(&ir)
43}
44
45#[cfg(test)]
46mod integration_tests {
47    use super::*;
48    use crate::encoders::algorithms::schema::types::{
49        FLAG_HAS_NULLS, FLAG_HAS_ROOT_KEY, FieldDef, FieldType, IntermediateRepresentation,
50        SchemaHeader, SchemaValue,
51    };
52    use parsers::{InputParser, JsonParser};
53    use serializers::{JsonSerializer, OutputSerializer};
54
55    #[test]
56    fn test_round_trip_simple() {
57        let fields = vec![
58            FieldDef::new("id", FieldType::U64),
59            FieldDef::new("name", FieldType::String),
60        ];
61        let header = SchemaHeader::new(2, fields);
62
63        let values = vec![
64            SchemaValue::U64(1),
65            SchemaValue::String("Alice".to_string()),
66            SchemaValue::U64(2),
67            SchemaValue::String("Bob".to_string()),
68        ];
69
70        let original = IntermediateRepresentation::new(header, values).unwrap();
71
72        // Pack and unpack
73        let packed = pack(&original);
74        let unpacked = unpack(&packed).unwrap();
75
76        assert_eq!(original, unpacked);
77    }
78
79    #[test]
80    fn test_round_trip_all_types() {
81        let fields = vec![
82            FieldDef::new("u64_field", FieldType::U64),
83            FieldDef::new("i64_field", FieldType::I64),
84            FieldDef::new("f64_field", FieldType::F64),
85            FieldDef::new("string_field", FieldType::String),
86            FieldDef::new("bool_field", FieldType::Bool),
87        ];
88        let header = SchemaHeader::new(1, fields);
89
90        let values = vec![
91            SchemaValue::U64(42),
92            SchemaValue::I64(-42),
93            SchemaValue::F64(std::f64::consts::PI),
94            SchemaValue::String("test".to_string()),
95            SchemaValue::Bool(true),
96        ];
97
98        let original = IntermediateRepresentation::new(header, values).unwrap();
99
100        let packed = pack(&original);
101        let unpacked = unpack(&packed).unwrap();
102
103        assert_eq!(original, unpacked);
104    }
105
106    #[test]
107    fn test_round_trip_with_root_key() {
108        let mut header = SchemaHeader::new(1, vec![FieldDef::new("id", FieldType::U64)]);
109        header.root_key = Some("users".to_string());
110        header.set_flag(FLAG_HAS_ROOT_KEY);
111
112        let values = vec![SchemaValue::U64(42)];
113        let original = IntermediateRepresentation::new(header, values).unwrap();
114
115        let packed = pack(&original);
116        let unpacked = unpack(&packed).unwrap();
117
118        assert_eq!(original, unpacked);
119    }
120
121    #[test]
122    fn test_round_trip_with_nulls() {
123        let mut header = SchemaHeader::new(
124            2,
125            vec![
126                FieldDef::new("id", FieldType::U64),
127                FieldDef::new("name", FieldType::String),
128            ],
129        );
130
131        // Mark second value as null (row 0, field 1)
132        let total_values: usize = 2 * 2; // 2 rows * 2 fields = 4 values
133        let bitmap_bytes = total_values.div_ceil(8); // 1 byte
134        let mut null_bitmap = vec![0u8; bitmap_bytes];
135        null_bitmap[0] |= 1 << 1; // Set bit 1 (second value)
136
137        header.null_bitmap = Some(null_bitmap);
138        header.set_flag(FLAG_HAS_NULLS);
139
140        let values = vec![
141            SchemaValue::U64(1),
142            SchemaValue::Null, // This is marked as null in bitmap
143            SchemaValue::U64(2),
144            SchemaValue::String("Bob".to_string()),
145        ];
146
147        let original = IntermediateRepresentation::new(header, values).unwrap();
148
149        let packed = pack(&original);
150        let unpacked = unpack(&packed).unwrap();
151
152        assert_eq!(original, unpacked);
153    }
154
155    #[test]
156    fn test_round_trip_array() {
157        let fields = vec![FieldDef::new(
158            "tags",
159            FieldType::Array(Box::new(FieldType::U64)),
160        )];
161        let header = SchemaHeader::new(1, fields);
162
163        let values = vec![SchemaValue::Array(vec![
164            SchemaValue::U64(1),
165            SchemaValue::U64(2),
166            SchemaValue::U64(3),
167        ])];
168
169        let original = IntermediateRepresentation::new(header, values).unwrap();
170
171        let packed = pack(&original);
172        let unpacked = unpack(&packed).unwrap();
173
174        assert_eq!(original, unpacked);
175    }
176
177    #[test]
178    fn test_round_trip_large_values() {
179        let fields = vec![
180            FieldDef::new("large_u64", FieldType::U64),
181            FieldDef::new("large_i64", FieldType::I64),
182        ];
183        let header = SchemaHeader::new(1, fields);
184
185        let values = vec![SchemaValue::U64(u64::MAX), SchemaValue::I64(i64::MIN)];
186
187        let original = IntermediateRepresentation::new(header, values).unwrap();
188
189        let packed = pack(&original);
190        let unpacked = unpack(&packed).unwrap();
191
192        assert_eq!(original, unpacked);
193    }
194
195    #[test]
196    fn test_round_trip_empty_string() {
197        let fields = vec![FieldDef::new("name", FieldType::String)];
198        let header = SchemaHeader::new(1, fields);
199
200        let values = vec![SchemaValue::String("".to_string())];
201
202        let original = IntermediateRepresentation::new(header, values).unwrap();
203
204        let packed = pack(&original);
205        let unpacked = unpack(&packed).unwrap();
206
207        assert_eq!(original, unpacked);
208    }
209
210    #[test]
211    fn test_round_trip_multiple_rows() {
212        let fields = vec![
213            FieldDef::new("id", FieldType::U64),
214            FieldDef::new("score", FieldType::F64),
215            FieldDef::new("active", FieldType::Bool),
216        ];
217        let header = SchemaHeader::new(3, fields);
218
219        let values = vec![
220            SchemaValue::U64(1),
221            SchemaValue::F64(95.5),
222            SchemaValue::Bool(true),
223            SchemaValue::U64(2),
224            SchemaValue::F64(87.3),
225            SchemaValue::Bool(false),
226            SchemaValue::U64(3),
227            SchemaValue::F64(92.1),
228            SchemaValue::Bool(true),
229        ];
230
231        let original = IntermediateRepresentation::new(header, values).unwrap();
232
233        let packed = pack(&original);
234        let unpacked = unpack(&packed).unwrap();
235
236        assert_eq!(original, unpacked);
237    }
238
239    #[test]
240    fn test_invalid_data() {
241        // Empty data
242        let result = unpack(&[]);
243        assert!(matches!(result, Err(SchemaError::UnexpectedEndOfData)));
244
245        // Truncated data
246        let result = unpack(&[0, 1, 2]);
247        assert!(result.is_err());
248    }
249
250    #[test]
251    fn test_json_full_roundtrip() {
252        let input = r#"{"users":[{"id":1,"name":"alice"},{"id":2,"name":"bob"}]}"#;
253        let ir = JsonParser::parse(input).unwrap();
254        let binary = pack(&ir);
255        let ir2 = unpack(&binary).unwrap();
256        let output = JsonSerializer::serialize(&ir2).unwrap();
257
258        // Parse both as serde_json::Value and compare (order-independent)
259        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
260        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
261        assert_eq!(input_value, output_value);
262    }
263
264    #[test]
265    fn test_json_simple_object() {
266        let input = r#"{"id":1,"name":"alice","score":95.5}"#;
267        let ir = JsonParser::parse(input).unwrap();
268        let binary = pack(&ir);
269        let ir2 = unpack(&binary).unwrap();
270        let output = JsonSerializer::serialize(&ir2).unwrap();
271
272        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
273        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
274        assert_eq!(input_value, output_value);
275    }
276
277    #[test]
278    fn test_json_nested_objects() {
279        let input = r#"{"user":{"profile":{"name":"alice","age":30}}}"#;
280        let ir = JsonParser::parse(input).unwrap();
281        let binary = pack(&ir);
282        let ir2 = unpack(&binary).unwrap();
283        let output = JsonSerializer::serialize(&ir2).unwrap();
284
285        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
286        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
287        assert_eq!(input_value, output_value);
288    }
289
290    #[test]
291    fn test_json_with_nulls() {
292        let input = r#"{"name":"alice","age":null,"active":true}"#;
293        let ir = JsonParser::parse(input).unwrap();
294        assert!(ir.header.has_flag(FLAG_HAS_NULLS));
295
296        let binary = pack(&ir);
297        let ir2 = unpack(&binary).unwrap();
298        let output = JsonSerializer::serialize(&ir2).unwrap();
299
300        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
301        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
302        assert_eq!(input_value, output_value);
303    }
304
305    #[test]
306    fn test_json_with_arrays() {
307        let input = r#"{"scores":[95,87,92],"tags":["rust","json"]}"#;
308        let ir = JsonParser::parse(input).unwrap();
309        let binary = pack(&ir);
310        let ir2 = unpack(&binary).unwrap();
311        let output = JsonSerializer::serialize(&ir2).unwrap();
312
313        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
314        let output_value: serde_json::Value = serde_json::from_str(&output).unwrap();
315        assert_eq!(input_value, output_value);
316    }
317
318    #[test]
319    fn test_encode_schema_roundtrip() {
320        let input = r#"{"users":[{"id":1,"name":"alice"},{"id":2,"name":"bob"}]}"#;
321        let encoded = encode_schema(input).unwrap();
322
323        // Validate frame delimiters
324        assert!(encoded.starts_with(frame::FRAME_START));
325        assert!(encoded.ends_with(frame::FRAME_END));
326
327        // Decode back to JSON
328        let decoded = decode_schema(&encoded).unwrap();
329
330        // Compare as JSON values (order-independent)
331        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
332        let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
333        assert_eq!(input_value, output_value);
334    }
335
336    #[test]
337    fn test_encode_schema_simple() {
338        let input = r#"{"id":1,"name":"alice","score":95.5}"#;
339        let encoded = encode_schema(input).unwrap();
340        let decoded = decode_schema(&encoded).unwrap();
341
342        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
343        let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
344        assert_eq!(input_value, output_value);
345    }
346
347    #[test]
348    fn test_encode_schema_with_nulls() {
349        let input = r#"{"name":"alice","age":null,"active":true}"#;
350        let encoded = encode_schema(input).unwrap();
351        let decoded = decode_schema(&encoded).unwrap();
352
353        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
354        let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
355        assert_eq!(input_value, output_value);
356    }
357
358    #[test]
359    fn test_encode_schema_empty_object() {
360        let input = r#"{}"#;
361        let result = encode_schema(input);
362        // Empty objects should fail or handle gracefully
363        // This depends on JsonParser behavior
364        println!("Empty object result: {:?}", result);
365    }
366
367    #[test]
368    fn test_decode_schema_invalid_frame() {
369        let invalid = "not_framed_data";
370        let result = decode_schema(invalid);
371        assert!(matches!(result, Err(SchemaError::InvalidFrame(_))));
372    }
373
374    #[test]
375    fn test_decode_schema_invalid_chars() {
376        let invalid = format!("{}ABC{}", frame::FRAME_START, frame::FRAME_END);
377        let result = decode_schema(&invalid);
378        assert!(matches!(result, Err(SchemaError::InvalidCharacter(_))));
379    }
380
381    #[test]
382    fn test_visual_wire_format() {
383        let input = r#"{"users":[{"id":1,"name":"alice"},{"id":2,"name":"bob"}]}"#;
384        let encoded = encode_schema(input).unwrap();
385
386        println!("\n=== Visual Wire Format ===");
387        println!("Input JSON: {}", input);
388        println!("Input length: {} bytes", input.len());
389        println!("\nEncoded output: {}", encoded);
390        println!(
391            "Encoded length: {} chars ({} bytes UTF-8)",
392            encoded.chars().count(),
393            encoded.len()
394        );
395
396        // Calculate compression ratio
397        let compression_ratio = input.len() as f64 / encoded.len() as f64;
398        println!("Compression ratio: {:.2}x", compression_ratio);
399
400        // Decode and verify
401        let decoded = decode_schema(&encoded).unwrap();
402        let input_value: serde_json::Value = serde_json::from_str(input).unwrap();
403        let output_value: serde_json::Value = serde_json::from_str(&decoded).unwrap();
404        assert_eq!(input_value, output_value);
405        println!("Roundtrip verified ✓\n");
406    }
407
408    #[test]
409    fn test_compression_comparison() {
410        let test_cases = [
411            r#"{"id":1}"#,
412            r#"{"id":1,"name":"alice"}"#,
413            r#"{"users":[{"id":1,"name":"alice"},{"id":2,"name":"bob"}]}"#,
414            r#"{"data":[1,2,3,4,5,6,7,8,9,10]}"#,
415        ];
416
417        println!("\n=== Compression Comparison ===");
418        for (i, input) in test_cases.iter().enumerate() {
419            let encoded = encode_schema(input).unwrap();
420            let ratio = input.len() as f64 / encoded.len() as f64;
421
422            println!(
423                "Test case {}: {} bytes → {} bytes ({:.2}x)",
424                i + 1,
425                input.len(),
426                encoded.len(),
427                ratio
428            );
429        }
430        println!();
431    }
432}
base_d/encoders/algorithms/schema/mod.rs

base_d/encoders/algorithms/schema/
mod.rs
