codec_rs/
lib.rs

1// SPDX-License-Identifier: MIT
2//! # codec-rs
3//!
4//! Rust port of the [Codec](https://github.com/wdunn001/Codec) binary
5//! transport protocol — the functional twin of `Codec.Net`,
6//! `@codecai/web`, and `codecai`.
7//!
8//! Codec carries `uint32` token IDs on the wire instead of UTF-8 / JSON,
9//! deferring text decoding to the presentation layer. This crate lets a
10//! Rust client decode/encode Codec frames, parse tokenizer maps,
11//! detokenize/tokenize, watch for tool calls, translate cross-vocab, and
12//! load maps over HTTP — all with full sha256 verification.
13//!
14//! ## Quick start
15//!
16//! ```no_run
17//! use codec_rs::{TokenizerMap, Detokenizer, DetokenizeOptions};
18//!
19//! # fn _example(json: &[u8], frames: Vec<codec_rs::CodecFrame>) {
20//! let map = TokenizerMap::from_json(json).unwrap();
21//! let mut detok = Detokenizer::new(&map);
22//! for frame in frames {
23//!     let opts = DetokenizeOptions { partial: !frame.done, render_special: false };
24//!     let text = detok.render(&frame.ids, opts);
25//!     print!("{text}");
26//! }
27//! # }
28//! ```
29//!
30//! See module-level docs and the project README for the full surface.
31
32pub mod byte_encoder;
33pub mod compression;
34pub mod detokenize;
35pub mod frame;
36pub mod longest_match;
37pub mod map;
38#[cfg(feature = "http")]
39pub mod map_loader;
40pub mod pretok_program;
41pub mod safety_policy;
42pub mod stream;
43pub mod tokenize;
44pub mod tool_watcher;
45pub mod translator;
46pub mod version_signaling;
47
48// Re-export the public surface.
49pub use version_signaling::{
50    parse_version_policy_document, parse_version_required, well_known_version_policy_url,
51    CodecVersionPolicyDocument, CodecVersionRequiredBody, HttpStatus, VersionSignalingError,
52    CODEC_CLIENT_VERSION, CODEC_CLIENT_VERSION_HEADER, CODEC_MIN_VERSION_HEADER,
53    CODEC_REQUIRED_FEATURES_HEADER,
54};
55#[cfg(feature = "http")]
56pub use version_signaling::discover_version_policy_blocking;
57pub use byte_encoder::{decode_byte_level_token, encode_byte_level_chars, METASPACE};
58pub use compression::{
59    hash_zstd_dict, select_zstd_dict_for_response, well_known_dict_url, CodecZstdDictError,
60    ZstdDictDiscoveryError,
61};
62#[cfg(feature = "http")]
63pub use compression::{discover_zstd_dict, discover_zstd_dict_blocking};
64pub use detokenize::{Detokenizer, DetokenizeOptions};
65pub use frame::{CodecFrame, IMapCache, MapCache, MemoryMapCache};
66pub use longest_match::{LongestMatchTokenizer, Tokenize};
67pub use map::{
68    TokenizerMap, TokenizerMapError, ToolCallingArgsFormat, ToolCallingBlock,
69    ToolCallingConvention, ToolCallingMarkers, ToolCallingResultFormat,
70};
71#[cfg(feature = "http")]
72pub use map_loader::{LoadError, LoadOptions, MapLoader, TokenizerMapHashMismatchError};
73pub use safety_policy::{
74    Category as SafetyCategory, CategoryAction, ClassifierBlock as SafetyClassifierBlock,
75    ClassifierHost, ClientHooksBlock as SafetyClientHooksBlock, EngineFeature,
76    PublisherBlock as SafetyPublisherBlock, RulesSummary as SafetyRulesSummary,
77    SafetyPolicyDescriptor, SafetyPolicyError, SafetyPolicyPointer,
78    POLICY_WELL_KNOWN_BASE,
79};
80#[cfg(feature = "http")]
81pub use safety_policy::{discover_safety_policy, load_safety_policy};
82pub use stream::{
83    decode_msgpack_stream, decode_protobuf_frame, decode_protobuf_stream, MsgpackFrameIter,
84    ProtobufFrameIter, StreamError,
85};
86pub use pretok_program::{run_pretok_program, PreTokOp, PreTokProgram};
87pub use tokenize::{BPETokenizer, ITokenizer};
88pub use tool_watcher::{ToolWatcher, ToolWatcherError, WatcherEvent, WatcherEventKind};
89pub use translator::{static_translation_table, translate_one_shot, Translator};
codec_rs/lib.rs

codec_rs/
lib.rs
