Expand description
KV Router - Radix tree data structures for LLM KV cache routing.
This crate provides the core radix tree implementation and protocols for efficient KV cache lookup and routing in distributed LLM inference systems.
Re-exports§
pub use indexer::concurrent_radix_tree;pub use indexer::concurrent_radix_tree_compressed;pub use indexer::positional as nested_map;pub use indexer::pruning as approx;pub use indexer::radix_tree;pub use scheduling::config;pub use scheduling::queue;pub use scheduling::selector;pub use sequences::multi_worker as multi_worker_sequence;pub use sequences::single as sequence;pub use self::multi_worker_sequence::ActiveSequencesMultiWorker;pub use self::multi_worker_sequence::ReplicaWorkerPolicy;pub use self::multi_worker_sequence::SequenceError;pub use self::multi_worker_sequence::SequencePublisher;pub use self::multi_worker_sequence::SequenceRequest;pub use self::multi_worker_sequence::SequenceSubscriber;pub use self::sequence::ActiveSequences;pub use self::sequence::RequestId;pub use self::sequences::PrefillTokenDeltas;pub use self::sequences::WorkerLoadProjection;pub use concurrent_radix_tree::ConcurrentRadixTree;pub use concurrent_radix_tree_compressed::ConcurrentRadixTreeCompressed;pub use config::KvRouterConfig;pub use config::RouterConfigOverride;pub use config::RouterPrefillLoadModel;pub use config::RouterQueuePolicy;pub use indexer::AnchorAwareBranchShardedIndexer;Deprecated pub use indexer::AnchorRef;pub use indexer::AnchorTask;pub use indexer::BranchShardedIndexer;pub use indexer::LowerTierContinuation;pub use indexer::LowerTierIndexer;pub use indexer::MaybeError;pub use indexer::SyncIndexer;pub use indexer::ThreadPoolIndexer;pub use nested_map::PositionalIndexer;pub use protocols::KvCacheEventError;pub use protocols::KvTransferEnforcement;pub use protocols::LocalBlockHash;pub use protocols::OverlapScores;pub use protocols::RouterEvent;pub use protocols::RouterEventSink;pub use protocols::WorkerConfigLike;pub use protocols::WorkerId;pub use protocols::compute_block_hash_for_seq;pub use queue::SchedulerQueue;pub use radix_tree::RadixTree;pub use scheduling::LocalScheduler;pub use scheduling::PrefillLoadEstimator;pub use scheduling::policy::FcfsPolicy;pub use scheduling::policy::RouterSchedulingPolicy;pub use scheduling::policy::SchedulingPolicy;pub use scheduling::policy::WsptPolicy;pub use scheduling::KvSchedulerError;pub use scheduling::PotentialLoad;pub use scheduling::SchedulingRequest;pub use scheduling::SchedulingResponse;pub use selector::DefaultWorkerSelector;pub use selector::WorkerSelector;