codec_rs/
lib.rs

1// SPDX-License-Identifier: MIT
2//! # codec-rs
3//!
4//! Rust port of the [Codec](https://github.com/wdunn001/Codec) binary
5//! transport protocol — the functional twin of `Codec.Net`,
6//! `@codecai/web`, and `codecai`.
7//!
8//! Codec carries `uint32` token IDs on the wire instead of UTF-8 / JSON,
9//! deferring text decoding to the presentation layer. This crate lets a
10//! Rust client decode/encode Codec frames, parse tokenizer maps,
11//! detokenize/tokenize, watch for tool calls, translate cross-vocab, and
12//! load maps over HTTP — all with full sha256 verification.
13//!
14//! ## Quick start
15//!
16//! ```no_run
17//! use codec_rs::{TokenizerMap, Detokenizer, DetokenizeOptions};
18//!
19//! # fn _example(json: &[u8], frames: Vec<codec_rs::CodecFrame>) {
20//! let map = TokenizerMap::from_json(json).unwrap();
21//! let mut detok = Detokenizer::new(&map);
22//! for frame in frames {
23//!     let opts = DetokenizeOptions { partial: !frame.done, render_special: false };
24//!     let text = detok.render(&frame.ids, opts);
25//!     print!("{text}");
26//! }
27//! # }
28//! ```
29//!
30//! See module-level docs and the project README for the full surface.
31
32pub mod byte_encoder;
33pub mod compression;
34pub mod detokenize;
35pub mod frame;
36pub mod longest_match;
37pub mod map;
38#[cfg(feature = "http")]
39pub mod map_loader;
40pub mod pretok_program;
41pub mod safety_policy;
42pub mod stream;
43pub mod tokenize;
44pub mod tool_watcher;
45pub mod translator;
46pub mod version_signaling;
47
48// Re-export the public surface.
49pub use version_signaling::{
50    parse_version_policy_document, parse_version_required, well_known_version_policy_url,
51    CodecVersionPolicyDocument, CodecVersionRequiredBody, HttpStatus, VersionSignalingError,
52    CODEC_CLIENT_VERSION, CODEC_CLIENT_VERSION_HEADER, CODEC_MIN_VERSION_HEADER,
53    CODEC_REQUIRED_FEATURES_HEADER,
54};
55#[cfg(feature = "http")]
56pub use version_signaling::discover_version_policy_blocking;
57pub use byte_encoder::{decode_byte_level_token, encode_byte_level_chars, METASPACE};
58pub use compression::{hash_zstd_dict, select_zstd_dict_for_response, CodecZstdDictError};
59pub use detokenize::{Detokenizer, DetokenizeOptions};
60pub use frame::{CodecFrame, IMapCache, MapCache, MemoryMapCache};
61pub use longest_match::{LongestMatchTokenizer, Tokenize};
62pub use map::{
63    TokenizerMap, TokenizerMapError, ToolCallingArgsFormat, ToolCallingBlock,
64    ToolCallingConvention, ToolCallingMarkers, ToolCallingResultFormat,
65};
66#[cfg(feature = "http")]
67pub use map_loader::{LoadError, LoadOptions, MapLoader, TokenizerMapHashMismatchError};
68pub use safety_policy::{
69    Category as SafetyCategory, CategoryAction, ClassifierBlock as SafetyClassifierBlock,
70    ClassifierHost, ClientHooksBlock as SafetyClientHooksBlock, EngineFeature,
71    PublisherBlock as SafetyPublisherBlock, RulesSummary as SafetyRulesSummary,
72    SafetyPolicyDescriptor, SafetyPolicyError, SafetyPolicyPointer,
73    POLICY_WELL_KNOWN_BASE,
74};
75#[cfg(feature = "http")]
76pub use safety_policy::{discover_safety_policy, load_safety_policy};
77pub use stream::{
78    decode_msgpack_stream, decode_protobuf_frame, decode_protobuf_stream, MsgpackFrameIter,
79    ProtobufFrameIter, StreamError,
80};
81pub use pretok_program::{run_pretok_program, PreTokOp, PreTokProgram};
82pub use tokenize::{BPETokenizer, ITokenizer};
83pub use tool_watcher::{ToolWatcher, ToolWatcherError, WatcherEvent, WatcherEventKind};
84pub use translator::{static_translation_table, translate_one_shot, Translator};
codec_rs/lib.rs

codec_rs/
lib.rs
