[package]
edition = "2024"
rust-version = "1.85"
name = "dx-serializer"
version = "0.1.0"
authors = ["DX Team <team@dx-www.dev>"]
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "A token-efficient serialization format for LLM context windows with high-performance binary encoding"
documentation = "https://docs.rs/dx-serializer"
readme = "README.md"
keywords = [
"serialization",
"parser",
"format",
"performance",
"zero-copy",
]
categories = [
"encoding",
"parser-implementations",
"asynchronous",
]
license = "MIT OR Apache-2.0"
repository = "https://github.com/dx-www/dx-www"
[package.metadata.wasm-pack.profile.release]
wasm-opt = false
[features]
arena = []
compression = [
"compression-lz4",
"compression-zstd",
]
compression-lz4 = ["lz4_flex"]
compression-zstd = ["zstd"]
converters = [
"serde",
"serde_json",
"serde_yaml",
"toml",
"bincode",
]
default = [
"converters",
"compression",
"serde-compat",
]
derive = ["dx-serializer-derive"]
full = [
"converters",
"compression",
"watch",
"token-counting",
"derive",
"parallel",
]
mmap = ["memmap2"]
parallel = ["rayon"]
serde-compat = [
"serde",
"bincode",
]
tiktoken = ["tiktoken-rs"]
token-counting = [
"tiktoken",
"tokenizers-hf",
]
tokenizers-hf = ["tokenizers"]
typed-cache = []
typed-cache-mmap = [
"typed-cache",
"mmap",
]
wasm = [
"wasm-bindgen",
"console_error_panic_hook",
"converters",
"compression",
]
watch = ["notify"]
[lib]
name = "serializer"
crate-type = ["rlib"]
path = "src/lib.rs"
[[bin]]
name = "dx-serialize"
path = "src/bin/serialize.rs"
[[example]]
name = "advanced"
path = "examples/advanced.rs"
[[example]]
name = "basic"
path = "examples/basic.rs"
[[example]]
name = "basic_usage"
path = "examples/basic_usage.rs"
[[example]]
name = "benchmark_dx_config"
path = "examples/benchmark_dx_config.rs"
[[example]]
name = "benchmark_vs_toon"
path = "examples/benchmark_vs_toon.rs"
[[example]]
name = "builder_demo"
path = "examples/builder_demo.rs"
[[example]]
name = "convert_all_to_human"
path = "examples/convert_all_to_human.rs"
[[example]]
name = "convert_package_json"
path = "examples/convert_package_json.rs"
[[example]]
name = "debug_human_parse"
path = "examples/debug_human_parse.rs"
[[example]]
name = "debug_tokens"
path = "examples/debug_tokens.rs"
[[example]]
name = "demo_all_converters"
path = "examples/demo_all_converters.rs"
[[example]]
name = "dx_machine_batch"
path = "examples/dx_machine_batch.rs"
[[example]]
name = "dx_playground_test"
path = "examples/dx_playground_test.rs"
[[example]]
name = "editor_workflow"
path = "examples/editor_workflow.rs"
[[example]]
name = "format_comparison"
path = "examples/format_comparison.rs"
[[example]]
name = "format_comparison_test"
path = "examples/format_comparison_test.rs"
[[example]]
name = "json_conversion"
path = "examples/json_conversion.rs"
[[example]]
name = "leaf_inline_demo"
path = "examples/leaf_inline_demo.rs"
[[example]]
name = "llm_to_human_format"
path = "examples/llm_to_human_format.rs"
[[example]]
name = "load_machine_files"
path = "examples/load_machine_files.rs"
[[example]]
name = "lsp"
path = "examples/lsp.rs"
[[example]]
name = "parallel_arena_demo"
path = "examples/parallel_arena_demo.rs"
[[example]]
name = "performance"
path = "examples/performance.rs"
[[example]]
name = "playground_benchmark"
path = "examples/playground_benchmark.rs"
[[example]]
name = "real_world_machine_test"
path = "examples/real_world_machine_test.rs"
[[example]]
name = "regenerate_dx"
path = "examples/regenerate_dx.rs"
[[example]]
name = "regenerate_human"
path = "examples/regenerate_human.rs"
[[example]]
name = "roundtrip_demo"
path = "examples/roundtrip_demo.rs"
[[example]]
name = "show_formats"
path = "examples/show_formats.rs"
[[example]]
name = "smart_keys_demo"
path = "examples/smart_keys_demo.rs"
[[example]]
name = "space_separator_test"
path = "examples/space_separator_test.rs"
[[example]]
name = "streaming"
path = "examples/streaming.rs"
[[example]]
name = "string_interning"
path = "examples/string_interning.rs"
[[example]]
name = "tables"
path = "examples/tables.rs"
[[example]]
name = "test_brutal"
path = "examples/test_brutal.rs"
[[example]]
name = "test_entry_order"
path = "examples/test_entry_order.rs"
[[example]]
name = "test_format_output"
path = "examples/test_format_output.rs"
[[example]]
name = "test_human_to_llm"
path = "examples/test_human_to_llm.rs"
[[example]]
name = "test_ideas"
path = "examples/test_ideas.rs"
[[example]]
name = "test_real"
path = "examples/test_real.rs"
[[example]]
name = "token_theory_test"
path = "examples/token_theory_test.rs"
[[example]]
name = "verify_claims"
path = "examples/verify_claims.rs"
[[test]]
name = "adaptive_optimization_test"
path = "tests/adaptive_optimization_test.rs"
[[test]]
name = "battle_hardening_property_tests"
path = "tests/battle_hardening_property_tests.rs"
[[test]]
name = "battle_hardening_tests"
path = "tests/battle_hardening_tests.rs"
[[test]]
name = "compression_test"
path = "tests/compression_test.rs"
[[test]]
name = "converter_tests"
path = "tests/converter_tests.rs"
[[test]]
name = "dx_format_spec"
path = "tests/dx_format_spec.rs"
[[test]]
name = "dx_machine_brutal_test"
path = "tests/dx_machine_brutal_test.rs"
[[test]]
name = "format_conversion_machine"
path = "tests/format_conversion_machine.rs"
[[test]]
name = "integration"
path = "tests/integration.rs"
[[test]]
name = "integration_converter"
path = "tests/integration_converter.rs"
[[test]]
name = "roundtrip_tests"
path = "tests/roundtrip_tests.rs"
[[test]]
name = "smart_key_tests"
path = "tests/smart_key_tests.rs"
[[test]]
name = "task_10_3_compact_syntax_serialization_props"
path = "tests/task_10_3_compact_syntax_serialization_props.rs"
[[test]]
name = "task_11_4_prefix_elimination_serialization_props"
path = "tests/task_11_4_prefix_elimination_serialization_props.rs"
[[test]]
name = "task_13_3_legacy_mode_props"
path = "tests/task_13_3_legacy_mode_props.rs"
[[test]]
name = "task_14_roundtrip_comprehensive"
path = "tests/task_14_roundtrip_comprehensive.rs"
[[test]]
name = "task_15_error_handling"
path = "tests/task_15_error_handling.rs"
[[test]]
name = "task_16_essence_examples"
path = "tests/task_16_essence_examples.rs"
[[test]]
name = "task_1_2_verification"
path = "tests/task_1_2_verification.rs"
[[test]]
name = "task_1_3_verification"
path = "tests/task_1_3_verification.rs"
[[test]]
name = "task_1_4_inline_object_props"
path = "tests/task_1_4_inline_object_props.rs"
[[test]]
name = "task_1_5_nested_array_props"
path = "tests/task_1_5_nested_array_props.rs"
[[test]]
name = "task_2_4_row_separator_props"
path = "tests/task_2_4_row_separator_props.rs"
[[test]]
name = "task_2_5_separator_autodetect_props"
path = "tests/task_2_5_separator_autodetect_props.rs"
[[test]]
name = "task_3_1_schema_separator_test"
path = "tests/task_3_1_schema_separator_test.rs"
[[test]]
name = "task_3_1_schema_serializer_test"
path = "tests/task_3_1_schema_serializer_test.rs"
[[test]]
name = "task_3_3_schema_parsing_props"
path = "tests/task_3_3_schema_parsing_props.rs"
[[test]]
name = "task_3_4_array_parsing_props"
path = "tests/task_3_4_array_parsing_props.rs"
[[test]]
name = "task_5_3_compact_syntax_props"
path = "tests/task_5_3_compact_syntax_props.rs"
[[test]]
name = "task_6_1_prefix_marker_parsing"
path = "tests/task_6_1_prefix_marker_parsing.rs"
[[test]]
name = "task_6_4_prefix_elimination_props"
path = "tests/task_6_4_prefix_elimination_props.rs"
[[test]]
name = "task_8_4_serializer_output_props"
path = "tests/task_8_4_serializer_output_props.rs"
[[test]]
name = "task_9_4_separator_selection_props"
path = "tests/task_9_4_separator_selection_props.rs"
[[test]]
name = "test_dots_preservation"
path = "tests/test_dots_preservation.rs"
[[test]]
name = "test_zstd"
path = "tests/test_zstd.rs"
[[test]]
name = "verify_converters"
path = "tests/verify_converters.rs"
[[test]]
name = "zero_integration"
path = "tests/zero_integration.rs"
[[bench]]
name = "adaptive_optimization"
path = "benches/adaptive_optimization.rs"
harness = false
[[bench]]
name = "arena_benchmark"
path = "benches/arena_benchmark.rs"
harness = false
[[bench]]
name = "arena_vs_rkyv"
path = "benches/arena_vs_rkyv.rs"
harness = false
[[bench]]
name = "brutal_truth"
path = "benches/brutal_truth.rs"
harness = false
[[bench]]
name = "comprehensive"
path = "benches/comprehensive.rs"
harness = false
[[bench]]
name = "compression_vs_pure_rkyv"
path = "benches/compression_vs_pure_rkyv.rs"
harness = false
[[bench]]
name = "dx_adaptive_vs_rkyv"
path = "benches/dx_adaptive_vs_rkyv.rs"
harness = false
[[bench]]
name = "dx_batch_vs_rkyv"
path = "benches/dx_batch_vs_rkyv.rs"
harness = false
[[bench]]
name = "dx_machine_complete"
path = "benches/dx_machine_complete.rs"
harness = false
[[bench]]
name = "dx_machine_standalone"
path = "benches/dx_machine_standalone.rs"
harness = false
[[bench]]
name = "dx_machine_vs_rkyv"
path = "benches/dx_machine_vs_rkyv.rs"
harness = false
[[bench]]
name = "dx_vs_rkyv"
path = "benches/dx_vs_rkyv.rs"
harness = false
[[bench]]
name = "dx_vs_rkyv_final"
path = "benches/dx_vs_rkyv_final.rs"
harness = false
[[bench]]
name = "dx_vs_rkyv_optimized"
path = "benches/dx_vs_rkyv_optimized.rs"
harness = false
[[bench]]
name = "dx_vs_rkyv_real"
path = "benches/dx_vs_rkyv_real.rs"
harness = false
[[bench]]
name = "dx_vs_toon"
path = "benches/dx_vs_toon.rs"
harness = false
[[bench]]
name = "dx_zero_bench"
path = "benches/dx_zero_bench.rs"
[[bench]]
name = "machine_format_compression_comparison"
path = "benches/machine_format_compression_comparison.rs"
harness = false
[[bench]]
name = "machine_format_perf"
path = "benches/machine_format_perf.rs"
harness = false
[[bench]]
name = "machine_vs_rkyv"
path = "benches/machine_vs_rkyv.rs"
harness = false
[[bench]]
name = "mmap_benchmark"
path = "benches/mmap_benchmark.rs"
harness = false
[[bench]]
name = "optimized_rkyv_complete"
path = "benches/optimized_rkyv_complete.rs"
harness = false
[[bench]]
name = "optimized_rkyv_test"
path = "benches/optimized_rkyv_test.rs"
harness = false
[[bench]]
name = "quick_comparison"
path = "benches/quick_comparison.rs"
harness = false
[[bench]]
name = "theme_compression_test"
path = "benches/theme_compression_test.rs"
harness = false
[[bench]]
name = "zerocopy_vs_rkyv"
path = "benches/zerocopy_vs_rkyv.rs"
harness = false
[[bench]]
name = "zstd_cache_roundtrip"
path = "benches/zstd_cache_roundtrip.rs"
[dependencies.bincode]
version = "1.3"
optional = true
[dependencies.blake3]
version = "1.5"
[dependencies.bytemuck]
version = "1.18"
features = ["derive"]
[dependencies.bytes]
version = "1.5"
[dependencies.console_error_panic_hook]
version = "0.1"
optional = true
[dependencies.dx-serializer-derive]
version = "0.1.0"
optional = true
[dependencies.hashbrown]
version = "0.15"
[dependencies.indexmap]
version = "2.7"
[dependencies.lz4_flex]
version = "0.11"
optional = true
[dependencies.memchr]
version = "2.7"
[dependencies.memmap2]
version = "0.9"
optional = true
[dependencies.notify]
version = "6.1"
optional = true
[dependencies.rayon]
version = "1.10"
optional = true
[dependencies.rkyv]
version = "0.8"
features = ["bytecheck"]
[dependencies.rustc-hash]
version = "1.1"
[dependencies.serde]
version = "1.0"
features = ["derive"]
optional = true
[dependencies.serde_json]
version = "1.0"
features = ["preserve_order"]
optional = true
[dependencies.serde_yaml]
version = "0.9"
optional = true
[dependencies.thiserror]
version = "2.0"
[dependencies.tiktoken-rs]
version = "0.6"
optional = true
[dependencies.tokenizers]
version = "0.21"
optional = true
[dependencies.toml]
version = "0.8"
optional = true
[dependencies.wasm-bindgen]
version = "=0.2.106"
optional = true
[dependencies.zstd]
version = "0.13"
optional = true
[dev-dependencies.bincode]
version = "1.3"
[dev-dependencies.criterion]
version = "0.5"
features = ["html_reports"]
[dev-dependencies.ctor]
version = "0.2"
[dev-dependencies.hex]
version = "0.4"
[dev-dependencies.postcard]
version = "1.0"
features = ["alloc"]
[dev-dependencies.proptest]
version = "1.4"
[dev-dependencies.rkyv]
version = "0.8"
features = ["bytecheck"]
[dev-dependencies.rkyv_derive]
version = "0.8"
[dev-dependencies.tempfile]
version = "3.14"
[target.'cfg(target_os = "linux")'.dependencies.io-uring]
version = "0.7"
optional = true
[target.'cfg(target_os = "linux")'.dependencies.libc]
version = "0.2"
[target.'cfg(target_os = "windows")'.dependencies.windows-sys]
version = "0.59"
features = [
"Win32_System_IO",
"Win32_Foundation",
"Win32_Storage_FileSystem",
]
optional = true
[lints.clippy]
all = "warn"
expect_used = "warn"
nursery = "warn"
pedantic = "warn"
unwrap_used = "warn"
[lints.rust]
missing_docs = "warn"
unsafe_code = "warn"