dx-serializer 0.1.0

A token-efficient serialization format for LLM context windows with high-performance binary encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

//! Task 3.1: Schema Parsing with Space and Comma Separators
//!
//! Validates Requirements 3.1, 3.2

use serializer::llm::parser::LlmParser;
use serializer::llm::types::DxLlmValue;

#[test]
fn test_space_separated_schema() {
    // Space-separated schema (new format)
    let input = "users:2(id name email)[\n1 Alice alice@ex.com\n2 Bob bob@ex.com\n]";
    let doc = LlmParser::parse(input).expect("Failed to parse space-separated schema");

    assert_eq!(doc.sections.len(), 1);
    let section = doc.sections.values().next().unwrap();
    assert_eq!(section.schema, vec!["id", "name", "email"]);
    assert_eq!(section.rows.len(), 2);

    // Verify first row
    assert_eq!(section.rows[0].len(), 3);
    assert_eq!(section.rows[0][0], DxLlmValue::Num(1.0));
    assert_eq!(section.rows[0][1], DxLlmValue::Str("Alice".to_string()));
    assert_eq!(
        section.rows[0][2],
        DxLlmValue::Str("alice@ex.com".to_string())
    );
}

#[test]
fn test_comma_separated_schema_backward_compat() {
    // Comma-separated schema (legacy format)
    let input = "users:2(id,name,email)[\n1 Alice alice@ex.com\n2 Bob bob@ex.com\n]";
    let doc = LlmParser::parse(input).expect("Failed to parse comma-separated schema");

    assert_eq!(doc.sections.len(), 1);
    let section = doc.sections.values().next().unwrap();
    assert_eq!(section.schema, vec!["id", "name", "email"]);
    assert_eq!(section.rows.len(), 2);

    // Verify first row
    assert_eq!(section.rows[0].len(), 3);
    assert_eq!(section.rows[0][0], DxLlmValue::Num(1.0));
    assert_eq!(section.rows[0][1], DxLlmValue::Str("Alice".to_string()));
    assert_eq!(
        section.rows[0][2],
        DxLlmValue::Str("alice@ex.com".to_string())
    );
}

#[test]
fn test_comma_separated_schema_with_spaces() {
    // Comma-separated with spaces (legacy format with whitespace)
    let input = "users:2(id, name, email)[\n1 Alice alice@ex.com\n2 Bob bob@ex.com\n]";
    let doc = LlmParser::parse(input).expect("Failed to parse comma-separated schema with spaces");

    assert_eq!(doc.sections.len(), 1);
    let section = doc.sections.values().next().unwrap();
    assert_eq!(section.schema, vec!["id", "name", "email"]);
    assert_eq!(section.rows.len(), 2);
}

#[test]
fn test_space_separated_schema_inline_rows() {
    // Space-separated schema with inline comma-separated rows
    let input =
        "users:3(id name email)[1 Alice alice@ex.com, 2 Bob bob@ex.com, 3 Carol carol@ex.com]";
    let doc =
        LlmParser::parse(input).expect("Failed to parse space-separated schema with inline rows");

    assert_eq!(doc.sections.len(), 1);
    let section = doc.sections.values().next().unwrap();
    assert_eq!(section.schema, vec!["id", "name", "email"]);
    assert_eq!(section.rows.len(), 3);
}

#[test]
fn test_single_column_space_schema() {
    // Single column schema (space-separated, but only one column)
    let input = "tags:2(name)[\nrust\npython\n]";
    let doc = LlmParser::parse(input).expect("Failed to parse single column schema");

    assert_eq!(doc.sections.len(), 1);
    let section = doc.sections.values().next().unwrap();
    assert_eq!(section.schema, vec!["name"]);
    assert_eq!(section.rows.len(), 2);
}

#[test]
fn test_many_columns_space_schema() {
    // Many columns with space-separated schema
    let input = "data:1(a b c d e f g h)[\n1 2 3 4 5 6 7 8\n]";
    let doc = LlmParser::parse(input).expect("Failed to parse many column schema");

    assert_eq!(doc.sections.len(), 1);
    let section = doc.sections.values().next().unwrap();
    assert_eq!(section.schema, vec!["a", "b", "c", "d", "e", "f", "g", "h"]);
    assert_eq!(section.rows.len(), 1);
    assert_eq!(section.rows[0].len(), 8);
}