Expand description
KV Router - Radix tree data structures for LLM KV cache routing.
This crate provides the core radix tree implementation and protocols for efficient KV cache lookup and routing in distributed LLM inference systems.
Re-exports§
pub use indexer::concurrent_radix_tree;pub use indexer::concurrent_radix_tree_compressed;pub use indexer::positional as nested_map;pub use indexer::pruning as approx;pub use indexer::radix_tree;pub use scheduling::config;pub use scheduling::queue;pub use scheduling::selector;pub use sequences::multi_worker as multi_worker_sequence;pub use sequences::single as sequence;pub use self::multi_worker_sequence::ActiveSequencesMultiWorker;pub use self::multi_worker_sequence::NoopSequencePublisher;pub use self::multi_worker_sequence::ReplicaWorkerPolicy;pub use self::multi_worker_sequence::SequenceError;pub use self::multi_worker_sequence::SequencePublisher;pub use self::multi_worker_sequence::SequenceRequest;pub use self::multi_worker_sequence::SequenceSubscriber;pub use self::sequence::ActiveSequences;pub use self::sequence::RequestId;pub use self::sequences::PrefillTokenDeltas;pub use self::sequences::WorkerLoadProjection;pub use concurrent_radix_tree::ConcurrentRadixTree;pub use concurrent_radix_tree_compressed::ConcurrentRadixTreeCompressed;pub use config::ConditionalDisaggPolicyKind;pub use config::KvRouterConfig;pub use config::RouterConfigOverride;pub use config::RouterPrefillLoadModel;pub use config::RouterQueuePolicy;pub use identity::DEFAULT_ROUTING_GROUP;pub use identity::DcId;pub use identity::RoutingPartitionId;pub use identity::RoutingPartitionRef;pub use indexer::AnchorAwareBranchShardedIndexer;Deprecated pub use indexer::AnchorRef;pub use indexer::AnchorTask;pub use indexer::BranchShardedIndexer;pub use indexer::LowerTierContinuation;pub use indexer::LowerTierIndexer;pub use indexer::MaybeError;pub use indexer::SyncIndexer;pub use indexer::ThreadPoolIndexer;pub use nested_map::PositionalIndexer;pub use protocols::KvCacheEventError;pub use protocols::KvTransferEnforcement;pub use protocols::LocalBlockHash;pub use protocols::OverlapScores;pub use protocols::RouterEvent;pub use protocols::RouterEventSink;pub use protocols::WorkerConfigLike;pub use protocols::WorkerId;pub use protocols::compute_block_hash_for_seq;pub use queue::SchedulerQueue;pub use radix_tree::RadixTree;pub use scheduling::LocalScheduler;pub use scheduling::PrefillLoadEstimator;pub use scheduling::policy::FcfsPolicy;pub use scheduling::policy::RouterSchedulingPolicy;pub use scheduling::policy::SchedulingPolicy;pub use scheduling::policy::WsptPolicy;pub use scheduling::KvSchedulerError;pub use scheduling::PotentialLoad;pub use scheduling::SchedulingRequest;pub use scheduling::SchedulingResponse;pub use selector::DefaultWorkerSelector;pub use selector::WorkerSelector;pub use tracking_hash::TrackingHashAlgorithm;pub use tracking_hash::TrackingHashContext;pub use tracking_hash::TrackingHashScope;
Modules§
- conditional_
disagg - Conditional-disagg bypass policy.
- identity
- Shared identities for routing partitions, logical KV indexers, and DC-local producer pools.
- indexer
- KV RadixTree
- protocols
- recovery
- scheduling
- sequences
- services
- Standalone services built from brokerless transport primitives.
- tracking_
hash - Router-owned hashing for derived active-sequence tracking state.
- zmq_
wire - Wire-format types for vLLM ZMQ KV event streams.