codec-rs 0.5.0

Isomorphic tokenizer + detokenizer for the Codec binary transport protocol — for Rust. Decodes streaming token IDs from Codec-compliant servers (vLLM, SGLang) and encodes text into IDs for the bidirectional path.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

// SPDX-License-Identifier: MIT
//! # codec-rs
//!
//! Rust port of the [Codec](https://github.com/wdunn001/Codec) binary
//! transport protocol — the functional twin of `Codec.Net`,
//! `@codecai/web`, and `codecai`.
//!
//! Codec carries `uint32` token IDs on the wire instead of UTF-8 / JSON,
//! deferring text decoding to the presentation layer. This crate lets a
//! Rust client decode/encode Codec frames, parse tokenizer maps,
//! detokenize/tokenize, watch for tool calls, translate cross-vocab, and
//! load maps over HTTP — all with full sha256 verification.
//!
//! ## Quick start
//!
//! ```no_run
//! use codec_rs::{TokenizerMap, Detokenizer, DetokenizeOptions};
//!
//! # fn _example(json: &[u8], frames: Vec<codec_rs::CodecFrame>) {
//! let map = TokenizerMap::from_json(json).unwrap();
//! let mut detok = Detokenizer::new(&map);
//! for frame in frames {
//!     let opts = DetokenizeOptions { partial: !frame.done, render_special: false };
//!     let text = detok.render(&frame.ids, opts);
//!     print!("{text}");
//! }
//! # }
//! ```
//!
//! See module-level docs and the project README for the full surface.

pub mod byte_encoder;
pub mod compression;
pub mod detokenize;
pub mod frame;
pub mod longest_match;
pub mod map;
#[cfg(feature = "http")]
pub mod map_loader;
pub mod pretok_program;
pub mod safety_policy;
pub mod stream;
pub mod tokenize;
pub mod tool_watcher;
pub mod translator;
pub mod version_signaling;

// Re-export the public surface.
pub use version_signaling::{
    parse_version_policy_document, parse_version_required, well_known_version_policy_url,
    CodecVersionPolicyDocument, CodecVersionRequiredBody, HttpStatus, VersionSignalingError,
    CODEC_CLIENT_VERSION, CODEC_CLIENT_VERSION_HEADER, CODEC_MIN_VERSION_HEADER,
    CODEC_REQUIRED_FEATURES_HEADER,
};
#[cfg(feature = "http")]
pub use version_signaling::discover_version_policy_blocking;
pub use byte_encoder::{decode_byte_level_token, encode_byte_level_chars, METASPACE};
pub use compression::{
    hash_zstd_dict, select_zstd_dict_for_response, well_known_dict_url, CodecZstdDictError,
    ZstdDictDiscoveryError,
};
#[cfg(feature = "http")]
pub use compression::{discover_zstd_dict, discover_zstd_dict_blocking};
pub use detokenize::{Detokenizer, DetokenizeOptions};
pub use frame::{CodecFrame, IMapCache, MapCache, MemoryMapCache};
pub use longest_match::{LongestMatchTokenizer, Tokenize};
pub use map::{
    TokenizerMap, TokenizerMapError, ToolCallingArgsFormat, ToolCallingBlock,
    ToolCallingConvention, ToolCallingMarkers, ToolCallingResultFormat,
};
#[cfg(feature = "http")]
pub use map_loader::{LoadError, LoadOptions, MapLoader, TokenizerMapHashMismatchError};
pub use safety_policy::{
    Category as SafetyCategory, CategoryAction, ClassifierBlock as SafetyClassifierBlock,
    ClassifierHost, ClientHooksBlock as SafetyClientHooksBlock, EngineFeature,
    PublisherBlock as SafetyPublisherBlock, RulesSummary as SafetyRulesSummary,
    SafetyPolicyDescriptor, SafetyPolicyError, SafetyPolicyPointer,
    POLICY_WELL_KNOWN_BASE,
};
#[cfg(feature = "http")]
pub use safety_policy::{discover_safety_policy, load_safety_policy};
pub use stream::{
    decode_msgpack_stream, decode_protobuf_frame, decode_protobuf_stream, MsgpackFrameIter,
    ProtobufFrameIter, StreamError,
};
pub use pretok_program::{run_pretok_program, PreTokOp, PreTokProgram};
pub use tokenize::{BPETokenizer, ITokenizer};
pub use tool_watcher::{ToolWatcher, ToolWatcherError, WatcherEvent, WatcherEventKind};
pub use translator::{static_translation_table, translate_one_shot, Translator};