dynamo-kv-router 1.3.0

KV Router - Radix tree for LLM KV cache routing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! KV Router - Radix tree data structures for LLM KV cache routing.
//!
//! This crate provides the core radix tree implementation and protocols for
//! efficient KV cache lookup and routing in distributed LLM inference systems.

mod active_set;
pub(crate) mod cleanup;
mod lookup_update;

pub mod indexer;
pub mod protocols;
pub mod recovery;
pub mod scheduling;
pub mod sequences;
pub mod services;
pub mod zmq_wire;

// Backward-compat re-exports: old top-level module paths still work
pub use indexer::concurrent_radix_tree;
pub use indexer::concurrent_radix_tree_compressed;
pub use indexer::positional as nested_map;
pub use indexer::pruning as approx;
pub use indexer::radix_tree;

pub use scheduling::config;
pub use scheduling::queue;
pub use scheduling::selector;
pub use sequences::multi_worker as multi_worker_sequence;
pub use sequences::single as sequence;

#[cfg(any(test, feature = "bench"))]
pub mod test_utils;

// Re-export key types for convenience
pub use self::multi_worker_sequence::{
    ActiveSequencesMultiWorker, ReplicaWorkerPolicy, SequenceError, SequencePublisher,
    SequenceRequest, SequenceSubscriber,
};
pub use self::sequence::{ActiveSequences, RequestId};
pub use self::sequences::{PrefillTokenDeltas, WorkerLoadProjection};
pub use concurrent_radix_tree::ConcurrentRadixTree;
pub use concurrent_radix_tree_compressed::ConcurrentRadixTreeCompressed;
pub use config::{
    KvRouterConfig, RouterConfigOverride, RouterPrefillLoadModel, RouterQueuePolicy,
    SharedCacheType,
};
#[allow(deprecated)]
pub use indexer::{
    AnchorAwareBranchShardedIndexer, AnchorRef, AnchorTask, BranchShardedIndexer,
    LowerTierContinuation, LowerTierIndexer, MaybeError, SharedKvCache, SyncIndexer,
    ThreadPoolIndexer,
};
pub use nested_map::PositionalIndexer;
pub use protocols::{
    KvCacheEventError, KvTransferEnforcement, LocalBlockHash, OverlapScores, RouterEvent,
    RouterEventSink, SharedCacheHits, WorkerConfigLike, WorkerId, compute_block_hash_for_seq,
};
pub use queue::SchedulerQueue;
pub use radix_tree::RadixTree;
pub use scheduling::LocalScheduler;
pub use scheduling::PrefillLoadEstimator;
pub use scheduling::policy::{FcfsPolicy, RouterSchedulingPolicy, SchedulingPolicy, WsptPolicy};
pub use scheduling::{KvSchedulerError, PotentialLoad, SchedulingRequest, SchedulingResponse};
pub use selector::{DefaultWorkerSelector, WorkerSelector};