dynamo-kv-router 1.4.0

KV Router - Radix tree for LLM KV cache routing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

pub mod config;
mod filter;
mod local;
pub mod overlap;
pub mod overlap_refresh;
pub mod policy;
pub mod policy_config;
pub mod policy_queue;
pub mod prefill_load;
pub mod queue;
mod queue_admission;
pub mod selector;

mod types;
pub use filter::*;
pub use local::LocalScheduler;
pub use overlap::{
    CacheHitEstimates, OverlapAnalysis, OverlapScoresResponse, OverlapSignals,
    SelectedWorkerTierSnapshot, SharedCacheOverlapScore, WorkerOverlapScore,
};
pub use overlap_refresh::{
    NoopOverlapScoresRefresh, OverlapScoresRefresh, RefreshedOverlap, TieredOverlapRefresher,
};
pub use policy_config::{
    PolicyClassConfig, PolicyProfile, RouterPolicyConfig, RouterPolicyConfigError,
};
pub use policy_queue::{
    PolicyQueue, PolicyQueueEntry, QueueLimitKind, QueueRejection, QueueSnapshot,
};
pub use prefill_load::{
    InvalidEffectivePrefillTokens, PrefillLoadEstimator, effective_prefill_tokens,
    prefill_load_hint_from_effective_tokens,
};
pub use queue::RequestLifecycleLease;
pub use queue_admission::{
    AdmissionAction, AdmissionDecision, AdmissionEvent, AdmissionId, AdmissionPolicyConfig,
    AdmissionRequest, PolicyClassAdmissionPolicies, PolicyClassAdmissionPolicy, RequestProgress,
    RequestProgressUpdater, WorkerEligibility, WorkerEligibilitySnapshot, WorkerPlacement,
};
pub use types::*;