clawft_kernel/workload_runtime/infer/
config.rs

1//! Adapter configuration: adopted or managed, and what managed needs.
2
3use std::path::{Path, PathBuf};
4use std::sync::Arc;
5use std::time::Duration;
6
7use super::spec::InferFlavor;
8use crate::model_manifest::ModelRegistry;
9
10/// Backoff for restarting a managed server that died while it should run.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub struct RestartPolicy {
13    /// Restarts before giving up (the operator then sees `Exited`).
14    pub max_restarts: u32,
15    /// Delay before the first restart; doubles each time.
16    pub base: Duration,
17    /// Longest delay.
18    pub cap: Duration,
19    /// How long a process must have been up (and answering) before the
20    /// restart count is cleared. A server that answers one probe and then
21    /// dies is still crash-looping and must reach `GaveUp`.
22    pub stable_after: Duration,
23}
24
25impl Default for RestartPolicy {
26    fn default() -> Self {
27        Self {
28            max_restarts: 5,
29            base: Duration::from_secs(2),
30            cap: Duration::from_secs(60),
31            stable_after: Duration::from_secs(60),
32        }
33    }
34}
35
36impl RestartPolicy {
37    /// Delay before restart number `attempt` (0-based).
38    pub fn delay(&self, attempt: u32) -> Duration {
39        self.base
40            .saturating_mul(1u32 << attempt.min(16))
41            .min(self.cap)
42    }
43}
44
45/// What managed mode needs.
46#[derive(Clone)]
47pub struct ManagedConfig {
48    /// The launcher script (`serve-llamacpp`, `serve`). Not used by Ollama,
49    /// which manages its own process.
50    pub serve_program: Option<PathBuf>,
51    /// Optional capability probe command (`llama-server --version`); exit 0
52    /// is a pass and the first stdout line is recorded as the version.
53    pub version_probe: Option<(PathBuf, Vec<String>)>,
54    /// Where per-instance working directories are created.
55    pub data_root: PathBuf,
56    /// Adopted model manifests; the only source of weight paths.
57    pub models: Arc<ModelRegistry>,
58    /// Extra environment for the server (the parent's is cleared).
59    pub env: Vec<(String, String)>,
60    /// Names of parent environment variables to forward (`HOME`,
61    /// `HF_HOME`, `PATH`). The default forwards the daemon's `PATH` and
62    /// `HOME` to the launcher: the server runs with the daemon's search
63    /// path and home directory. A forwarded or explicit `env` `PATH` wins
64    /// over the supervisor's default one.
65    pub env_passthrough: Vec<String>,
66    /// `(uid, gid)` to drop to when the host runs as root.
67    pub run_as: Option<(u32, u32)>,
68    /// Restart backoff.
69    pub restart: RestartPolicy,
70    /// Ollama `keep_alive` sent when loading a model (`5m`, `30m`, `-1`).
71    pub keep_alive: String,
72    /// Unified-memory budget and co-residency, shared by every managed
73    /// adapter on the node. `None`: no check (a single adapter, or the
74    /// operator did not ask for one).
75    pub ledger: Option<Arc<super::residency::ResidencyLedger>>,
76}
77
78impl ManagedConfig {
79    /// Defaults: no launcher, `HOME` and `PATH` forwarded, default backoff.
80    pub fn new(models: Arc<ModelRegistry>, data_root: PathBuf) -> Self {
81        Self {
82            serve_program: None,
83            version_probe: None,
84            data_root,
85            models,
86            env: Vec::new(),
87            env_passthrough: vec!["HOME".into(), "PATH".into()],
88            run_as: None,
89            restart: RestartPolicy::default(),
90            keep_alive: "30m".into(),
91            ledger: None,
92        }
93    }
94
95    /// Share a residency ledger (budget and co-residency).
96    pub fn with_ledger(mut self, ledger: Arc<super::residency::ResidencyLedger>) -> Self {
97        self.ledger = Some(ledger);
98        self
99    }
100
101    /// Launcher script.
102    pub fn with_serve_program(mut self, p: impl Into<PathBuf>) -> Self {
103        self.serve_program = Some(p.into());
104        self
105    }
106}
107
108/// The model lab's launcher for `flavor` under `home` (`<home>/llm/bin/...`).
109/// Only builds the path; nothing is read or checked.
110pub fn lab_serve_program(home: &Path, flavor: InferFlavor) -> Option<PathBuf> {
111    let name = match flavor {
112        InferFlavor::LlamaCpp => "serve-llamacpp",
113        InferFlavor::MlxLm => "serve",
114        InferFlavor::Ollama => return None,
115    };
116    Some(home.join("llm").join("bin").join(name))
117}
118
119/// Observe-only, or launch and control.
120///
121/// Adopted mode is observe-only. Everything it does is a read against a
122/// loopback address taken from the spec, and that includes `admit`, which
123/// the host calls before the governance gate (an admission self-check is
124/// read-only by contract), so a spec's loopback port is probed with GETs
125/// before any permit is consulted. [`InferConfig::allowed_ports`] bounds
126/// which ports that can be.
127#[derive(Clone)]
128pub enum InferMode {
129    /// Register and health-check a server already running; never start,
130    /// stop or signal it.
131    Adopted,
132    /// Start, stop, restart and unload the server.
133    Managed(Box<ManagedConfig>),
134}
135
136/// One adapter's configuration.
137#[derive(Clone)]
138pub struct InferConfig {
139    /// Server software.
140    pub flavor: InferFlavor,
141    /// Adopted or managed.
142    pub mode: InferMode,
143    /// Per-request timeout of probes.
144    pub probe_timeout: Duration,
145    /// Port the capability probe asks (the flavor's conventional port when
146    /// absent). Instances carry their own ports.
147    pub probe_port: Option<u16>,
148    /// Inclusive port range instances may use. Checked before anything is
149    /// probed: a spec naming a port outside it never causes a connection.
150    pub allowed_ports: Option<(u16, u16)>,
151    /// Timeout of [`InferRuntime::deep_health`](super::InferRuntime::deep_health)
152    /// (a cold model can take minutes to answer its first token).
153    pub deep_timeout: Duration,
154}
155
156impl InferConfig {
157    /// Limit instances to ports in `lo..=hi`.
158    pub fn with_port_range(mut self, lo: u16, hi: u16) -> Self {
159        self.allowed_ports = Some((lo, hi));
160        self
161    }
162
163    /// Probe a different port for capabilities.
164    pub fn with_probe_port(mut self, port: u16) -> Self {
165        self.probe_port = Some(port);
166        self
167    }
168
169    /// An adopting adapter.
170    pub fn adopted(flavor: InferFlavor) -> Self {
171        Self {
172            flavor,
173            mode: InferMode::Adopted,
174            probe_timeout: Duration::from_secs(2),
175            probe_port: None,
176            allowed_ports: None,
177            deep_timeout: Duration::from_secs(300),
178        }
179    }
180
181    /// A managing adapter.
182    pub fn managed(flavor: InferFlavor, cfg: ManagedConfig) -> Self {
183        Self {
184            flavor,
185            mode: InferMode::Managed(Box::new(cfg)),
186            probe_timeout: Duration::from_secs(2),
187            probe_port: None,
188            allowed_ports: None,
189            deep_timeout: Duration::from_secs(300),
190        }
191    }
192}
clawft_kernel/workload_runtime/infer/config.rs

clawft_kernel/workload_runtime/infer/
config.rs
