dynamo-llm 1.5.0

Dynamo LLM Library
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

mod model;
pub(crate) mod readiness;
pub(crate) use model::GenerateEngineSelection;
pub use model::Model;

pub mod kv_source_membership;
pub use kv_source_membership::{
    KvEventSource, KvSourceAdvertisement, KvSourceAmbiguity, KvSourceId, KvSourceKey,
    KvSourceMembership, KvSourceMembershipError, KvSourceMembershipView, KvSourceStatus,
    KvSourceTransition, KvStateEndpointResolution, PublisherId, resolve_kv_state_endpoint,
};

mod kv_source_watch;
pub(crate) use kv_source_watch::KvSourceMembershipCoordinator;
pub use kv_source_watch::KvSourceMembershipWatch;

pub mod kv_state_agent;

mod model_manager;
pub use model_manager::{ModelManager, ModelManagerError, UNKNOWN_METRIC_MODEL};

mod controller;

mod allocator;

mod worker_set;
pub use worker_set::WorkerSet;
pub(crate) use worker_set::{CommittedWorkerSetTarget, WorkerSetTarget, WorkerSetTargetId};

pub(crate) mod runtime_configs;
pub use runtime_configs::{RuntimeConfigWatch, runtime_config_watch};

mod endpoint_card;
pub use endpoint_card::wait_for_endpoint_model_card;

mod watcher;
pub use watcher::{ModelUpdate, ModelWatcher};

mod worker_monitor;
pub use worker_monitor::{
    KvWorkerMonitor, LoadThresholdConfig, LoadThresholdHandle, WORKER_TYPE_DECODE,
    WORKER_TYPE_PREFILL, WorkerLoadState,
};