dynamo_llm/
entrypoint.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! The entrypoint module provides tools to build a Dynamo runner.
5//! - Create an EngineConfig of the engine (potentially auto-discovered) to execute
6//! - Connect it to an Input
7
8pub mod input;
9pub use input::{build_routed_pipeline, build_routed_pipeline_with_preprocessor};
10
11use std::sync::Arc;
12
13use dynamo_runtime::pipeline::RouterMode;
14
15use crate::{
16    backend::ExecutionContext, engines::StreamingEngine, kv_router::KvRouterConfig,
17    local_model::LocalModel,
18};
19
20#[derive(Debug, Clone, Default)]
21pub struct RouterConfig {
22    pub router_mode: RouterMode,
23    pub kv_router_config: KvRouterConfig,
24    pub busy_threshold: Option<f64>,
25}
26
27impl RouterConfig {
28    pub fn new(router_mode: RouterMode, kv_router_config: KvRouterConfig) -> Self {
29        Self {
30            router_mode,
31            kv_router_config,
32            busy_threshold: None,
33        }
34    }
35
36    pub fn with_busy_threshold(mut self, threshold: Option<f64>) -> Self {
37        self.busy_threshold = threshold;
38        self
39    }
40}
41
42#[derive(Clone)]
43pub enum EngineConfig {
44    /// Remote networked engines that we discover via etcd
45    Dynamic(Box<LocalModel>),
46
47    /// Remote networked engines that we know about at startup
48    StaticRemote(Box<LocalModel>),
49
50    /// A Full service engine does it's own tokenization and prompt formatting.
51    StaticFull {
52        engine: Arc<dyn StreamingEngine>,
53        model: Box<LocalModel>,
54        is_static: bool,
55    },
56
57    /// A core engine expects to be wrapped with pre/post processors that handle tokenization.
58    StaticCore {
59        engine: ExecutionContext,
60        model: Box<LocalModel>,
61        is_static: bool,
62    },
63}
64
65impl EngineConfig {
66    fn local_model(&self) -> &LocalModel {
67        use EngineConfig::*;
68        match self {
69            Dynamic(lm) => lm,
70            StaticRemote(lm) => lm,
71            StaticFull { model, .. } => model,
72            StaticCore { model, .. } => model,
73        }
74    }
75}
dynamo_llm/entrypoint.rs

dynamo_llm/
entrypoint.rs
