serializer/llm/
mod.rs

1//! DX Serializer LLM Format Module
2//!
3//! This module provides support for multiple interconvertible formats:
4//!
5//! ## DX Serializer Format (Token-Efficient LLM Format)
6//!
7//! A token-efficient serialization format optimized for LLMs:
8//! - `key=value` for simple key-value pairs
9//! - `name(key=val key2=val2)` for objects (parentheses, space-separated)
10//! - `name[col1 col2 col3](rows)` for tables (wrapped dataframe format)
11//! - `key=[item1 item2 item3]` for arrays (square brackets, space-separated)
12//! - `true`/`false` for booleans, `null` for null values
13//! - Use quotes `"..."` for multi-word strings
14//!
15//! ## Human Format (Clean TOML-like)
16//!
17//! Clean, hand-editable format:
18//! - `=` for single scalar values
19//! - `:` + `-` lines for arrays
20//! - `#` for comments
21//! - Nested sections like `[stack.js]`
22//!
23//! ## Machine Format
24//!
25//! Binary format for runtime using RKYV's zero-copy architecture.
26//!
27//! The architecture follows a "hub and spoke" model where all formats convert through
28//! a common internal representation (`DxDocument`), ensuring consistent round-trip behavior.
29
30pub mod abbrev;
31pub mod cache_generator;
32pub mod convert;
33pub mod human_formatter;
34pub mod human_parser;
35pub mod machine_zerocopy;
36pub mod parser;
37pub mod pretty_printer;
38pub mod section_names;
39pub mod serializer;
40pub mod serializer_output;
41pub mod table_wrapper;
42pub mod tokens;
43pub mod types;
44
45#[cfg(test)]
46mod abbrev_props;
47#[cfg(test)]
48mod convert_props;
49#[cfg(test)]
50mod human_props;
51#[cfg(test)]
52mod llm_props;
53
54// Re-export main types
55pub use abbrev::AbbrevDict;
56pub use cache_generator::{CacheConfig, CacheError, CacheGenerator, CachePaths, CacheResult};
57#[cfg(feature = "mmap")]
58pub use convert::machine_file_to_document_mmap;
59pub use convert::{
60    ConvertError, MachineFormat, document_to_human, document_to_llm, document_to_machine,
61    human_to_document, human_to_llm, human_to_machine, human_to_machine_uncompressed,
62    is_llm_format, llm_to_document, llm_to_human, llm_to_machine, machine_bytes_to_document,
63    machine_to_document, machine_to_human, machine_to_llm,
64    try_document_to_machine_with_compression,
65};
66pub use human_formatter::{HumanFormatConfig, HumanFormatter};
67pub use human_parser::{HumanParseError, HumanParser};
68pub use machine_zerocopy::{ZeroCopyDocument, ZeroCopyError, ZeroCopyMachine};
69pub use parser::{LlmParser, ParseError};
70pub use pretty_printer::{PrettyPrintError, PrettyPrinter, PrettyPrinterConfig};
71pub use section_names::SectionNameDict;
72pub use serializer::{LlmSerializer, SerializerConfig};
73pub use serializer_output::{
74    SerializerOutput, SerializerOutputConfig, SerializerOutputError, SerializerPaths,
75    SerializerResult,
76};
77pub use table_wrapper::{TableWrapper, TableWrapperConfig};
78pub use tokens::{ModelType, TokenCounter, TokenInfo};
79pub use types::{DxDocument, DxLlmValue, DxSection};
serializer/llm/mod.rs

serializer/llm/
mod.rs
