use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use crate::error::{NewtError, Result};
use crate::router::Tier;
use super::dropin::{
classify_untagged_dropin, disk_record_tag, parse_probe_record, DropinOwner, ProbeObservation,
};
use super::{
backend_is_routable, expand_tilde, pin_requested_selection, select_backend_slot, Config,
SlotSelection,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "lowercase")]
pub enum BackendKind {
#[default]
Ollama,
#[serde(alias = "vllm", alias = "openai-compatible")]
Openai,
Embedded,
#[serde(alias = "claude")]
Anthropic,
}
impl BackendKind {
pub fn label(self) -> &'static str {
match self {
Self::Ollama => "ollama",
Self::Openai => "openai",
Self::Embedded => "embedded",
Self::Anthropic => "anthropic",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Engine {
Ollama,
#[serde(alias = "llama-cpp", alias = "llama.cpp")]
LlamaCpp,
Vllm,
}
impl Engine {
pub fn label(self) -> &'static str {
match self {
Self::Ollama => "ollama",
Self::LlamaCpp => "llama.cpp",
Self::Vllm => "vllm",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum OpenAiApi {
#[default]
#[serde(alias = "chat", alias = "completions")]
ChatCompletions,
Responses,
}
impl OpenAiApi {
pub fn label(self) -> &'static str {
match self {
Self::ChatCompletions => "chat_completions",
Self::Responses => "responses",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Serving {
Multiplexer,
Instance,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ManagedMode {
Shared,
Dedicated,
}
pub fn derive_serving(kind: BackendKind, served_count: usize) -> Serving {
match kind {
BackendKind::Ollama => Serving::Multiplexer,
BackendKind::Openai => {
if served_count == 1 {
Serving::Instance
} else {
Serving::Multiplexer
}
}
BackendKind::Embedded => Serving::Instance,
BackendKind::Anthropic => Serving::Multiplexer,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(super) enum RecordTag {
OperatorV1,
ProbeV1,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct BackendProvenance {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub probed: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub derived_serving: Option<bool>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct BackendConfig {
#[serde(default)]
pub name: String,
#[serde(default)]
pub endpoint: String,
#[serde(
default = "default_backend_slots",
skip_serializing_if = "backend_slots_are_default"
)]
pub slots: BackendSlots,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model_path: Option<String>,
#[serde(default)]
pub tiers: Vec<Tier>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub kind: Option<BackendKind>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub api: Option<OpenAiApi>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub api_key_file: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub api_key_env: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub serving: Option<Serving>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub managed: Option<ManagedMode>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub engine: Option<Engine>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub host: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub coexist: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ram_gib: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub card: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capability: Option<crate::model_card::Capability>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provenance: Option<BackendProvenance>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(transparent)]
pub struct BackendSlots(std::num::NonZeroUsize);
impl BackendSlots {
#[must_use]
pub const fn get(self) -> usize {
self.0.get()
}
}
impl Default for BackendSlots {
fn default() -> Self {
Self(std::num::NonZeroUsize::MIN)
}
}
fn default_backend_slots() -> BackendSlots {
BackendSlots::default()
}
fn backend_slots_are_default(slots: &BackendSlots) -> bool {
*slots == default_backend_slots()
}
impl BackendConfig {
pub fn with_dropin_edits(text: &str, edits: &[(&str, Option<String>)]) -> Result<String> {
let mut doc = text
.parse::<toml_edit::DocumentMut>()
.map_err(|e| NewtError::Config(format!("backend drop-in is not valid TOML: {e}")))?;
let root = doc.as_table_mut();
for (key, value) in edits {
match value {
Some(new) => match root.get_mut(key) {
Some(item) => {
let decor = item.as_value().map(|value| value.decor().clone());
*item = toml_edit::value(new.as_str());
if let (Some(decor), Some(value)) = (decor, item.as_value_mut()) {
*value.decor_mut() = decor;
}
}
None => {
root.insert(key, toml_edit::value(new.as_str()));
}
},
None => {
root.remove(key);
}
}
}
Ok(doc.to_string())
}
#[must_use]
pub fn chat_completions_capability(&self) -> crate::model_card::ChatCompletionsCapability {
self.capability
.as_ref()
.and_then(|capability| capability.chat_completions)
.unwrap_or_default()
}
#[must_use]
pub fn emits_leading_reasoning(&self) -> bool {
self.capability
.as_ref()
.and_then(|capability| capability.emits_leading_reasoning)
.unwrap_or(false)
}
#[must_use]
pub fn reasoning_replay_scope(&self) -> crate::model_card::ReasoningReplayScope {
self.capability
.as_ref()
.and_then(|capability| capability.reasoning_replay_scope)
.unwrap_or_default()
}
pub fn effective_model(&self) -> Option<&str> {
self.model.as_deref().filter(|m| !m.trim().is_empty())
}
pub fn needs_kind_probe(&self) -> bool {
self.kind.is_none()
}
pub fn kind_label(&self) -> &'static str {
self.kind.map(BackendKind::label).unwrap_or("auto")
}
pub fn resolve_api_key(&self) -> Option<String> {
match self.resolve_api_key_detailed() {
Ok(v) => v,
Err(e) => {
crate::secrets::warn_once(self.api_key_file.as_deref().unwrap_or(&self.name), &e);
None
}
}
}
pub fn resolve_api_key_detailed(
&self,
) -> std::result::Result<Option<String>, crate::secrets::SecretsError> {
resolve_api_key_common(self.api_key_env.as_deref(), self.api_key_file.as_deref())
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct BackendRef {
#[serde(skip_serializing_if = "Option::is_none")]
pub endpoint: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub kind: Option<BackendKind>,
#[serde(skip_serializing_if = "Option::is_none")]
pub api_key_env: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub api_key_file: Option<String>,
}
impl BackendRef {
#[must_use]
pub fn pins_a_different_host(&self) -> bool {
self.endpoint.is_some()
}
#[must_use]
pub fn resolve(
&self,
session_endpoint: &str,
session_model: &str,
session_kind: BackendKind,
session_key: &Option<String>,
) -> (String, String, BackendKind, Option<String>) {
let own_key =
resolve_api_key_common(self.api_key_env.as_deref(), self.api_key_file.as_deref())
.unwrap_or_default();
let key = if self.pins_a_different_host() {
own_key
} else {
own_key.or_else(|| session_key.clone())
};
(
self.endpoint
.clone()
.unwrap_or_else(|| session_endpoint.to_string()),
self.model
.clone()
.unwrap_or_else(|| session_model.to_string()),
self.kind.unwrap_or(session_kind),
key,
)
}
}
pub(crate) fn resolve_api_key_common(
api_key_env: Option<&str>,
api_key_file: Option<&str>,
) -> std::result::Result<Option<String>, crate::secrets::SecretsError> {
if let Some(var) = api_key_env {
if let Ok(val) = std::env::var(var) {
let val = val.trim();
if !val.is_empty() {
return Ok(Some(val.to_string()));
}
}
}
if let Some(path) = api_key_file {
let expanded = expand_tilde(path);
return crate::secrets::resolve_token_file(&expanded);
}
Ok(None)
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct BackendDestination {
pub endpoint: Option<String>,
pub model_path: Option<String>,
}
impl BackendDestination {
#[must_use]
pub fn new(endpoint: Option<String>, model_path: Option<String>) -> Self {
Self {
endpoint: endpoint.filter(|e| !e.is_empty()),
model_path: model_path.filter(|p| !p.is_empty()),
}
}
#[must_use]
pub fn of(backend: &BackendConfig) -> Self {
Self::new(Some(backend.endpoint.clone()), backend.model_path.clone())
}
#[must_use]
pub fn is_concrete(&self) -> bool {
let endpoint = self.endpoint.as_deref().is_some_and(|e| !e.is_empty());
let model_path = self.model_path.as_deref().is_some_and(|p| !p.is_empty());
endpoint ^ model_path
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct DeclaredBackend {
pub destination: BackendDestination,
pub model: Option<String>,
pub card: Option<String>,
pub serving: Option<Serving>,
pub kind: Option<BackendKind>,
pub api: Option<OpenAiApi>,
pub managed: Option<ManagedMode>,
}
impl DeclaredBackend {
#[must_use]
pub fn of(backend: &BackendConfig) -> Self {
Self {
destination: BackendDestination::of(backend),
model: backend.effective_model().map(str::to_string),
card: backend.card.clone(),
serving: backend.serving,
kind: backend.kind,
api: backend.api,
managed: backend.managed,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RequestMode {
ExclusiveDestination,
FieldOnly,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BackendRequest {
pub mode: RequestMode,
pub endpoint: Option<String>,
pub model_path: Option<String>,
pub model: Option<String>,
pub card: Option<String>,
pub serving: Option<Serving>,
pub kind: Option<BackendKind>,
pub api: Option<OpenAiApi>,
}
impl BackendRequest {
fn from_override(over: &BackendOverride) -> Self {
let mode = if over.endpoint.is_some() || over.model_path.is_some() {
RequestMode::ExclusiveDestination
} else {
RequestMode::FieldOnly
};
Self {
mode,
endpoint: over.endpoint.clone().filter(|e| !e.is_empty()),
model_path: over.model_path.clone().filter(|p| !p.is_empty()),
model: over.model.clone().filter(|m| !m.trim().is_empty()),
card: over.card.clone(),
serving: over.serving,
kind: over.kind,
api: over.api,
}
}
#[must_use]
pub fn destination_over(&self, declared: &BackendDestination) -> BackendDestination {
if self.endpoint.is_none() && self.model_path.is_none() {
return declared.clone();
}
BackendDestination {
endpoint: self.endpoint.clone(),
model_path: self.model_path.clone(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BackendResolutionReceipt {
pub declaration: DeclaredBackend,
pub request: Option<BackendRequest>,
pub observation: Option<ProbeObservation>,
pub binding: crate::model_card::CardBindingSeed,
}
pub(super) fn validate_backend_names<'a>(
backends: impl Iterator<Item = &'a BackendConfig>,
) -> std::result::Result<(), String> {
let mut seen = std::collections::BTreeSet::new();
for (i, b) in backends.enumerate() {
if b.name.trim().is_empty() {
return Err(format!(
"backend #{} has no name — every [[backends]] entry needs a unique \
`name` (selection, overrides, and card bindings are name-based)",
i + 1
));
}
if !seen.insert(b.name.clone()) {
return Err(format!(
"two backends share the name `{}` — backend selection is name-based \
everywhere (default_backend, $NEWT_PROVIDER, --backend-*, card \
bindings), so a duplicate can activate the wrong card; rename one",
b.name
));
}
}
Ok(())
}
fn backend_has_destination(b: &BackendConfig) -> bool {
!b.endpoint.is_empty() || b.model_path.as_deref().is_some_and(|p| !p.is_empty())
}
fn validate_backend_destination(b: &BackendConfig) -> std::result::Result<(), String> {
if !b.endpoint.is_empty() && b.model_path.as_deref().is_some_and(|p| !p.is_empty()) {
return Err(format!(
"backend `{}` declares BOTH an endpoint and a model_path — a backend has \
ONE destination; remove one",
b.name
));
}
Ok(())
}
enum NameMatch {
Missing,
Unique(usize),
Ambiguous,
}
#[derive(Debug)]
struct AssemblySlot {
declaration: BackendConfig,
observation: Option<ProbeObservation>,
request: Option<BackendOverride>,
}
impl AssemblySlot {
fn declared(declaration: BackendConfig) -> Self {
Self {
declaration,
observation: None,
request: None,
}
}
fn observed_view(&self) -> BackendConfig {
let mut backend = self.declaration.clone();
if let Some(obs) = &self.observation {
overlay_observation(&mut backend, obs);
}
normalize_destination_kind(&mut backend);
backend
}
}
fn normalize_destination_kind(backend: &mut BackendConfig) {
if backend.endpoint.is_empty() && backend.model_path.as_deref().is_some_and(|p| !p.is_empty()) {
backend.kind = Some(BackendKind::Embedded);
backend.serving = Some(Serving::Instance);
} else if !backend.endpoint.is_empty() && backend.kind == Some(BackendKind::Embedded) {
backend.kind = None;
}
}
fn overlay_observation(backend: &mut BackendConfig, obs: &ProbeObservation) {
if let Some(kind) = obs.kind {
backend.kind = Some(kind);
}
if let Some(api) = obs.api {
backend.api = Some(api);
}
let (serving, model) = obs.serving_axis();
if let Some(serving) = serving {
backend.serving = Some(serving);
if let Some(model) = model {
backend.model = Some(model);
}
}
}
#[derive(Debug)]
struct PendingProbe {
path: PathBuf,
stem: String,
observation: ProbeObservation,
}
#[derive(Debug)]
pub(super) struct BackendAssembly {
slots: Vec<AssemblySlot>,
pending_probes: Vec<PendingProbe>,
operator_configured: bool,
requested: bool,
warnings: Vec<String>,
}
fn emitted_warnings() -> &'static std::sync::Mutex<std::collections::HashSet<String>> {
static EMITTED: std::sync::OnceLock<std::sync::Mutex<std::collections::HashSet<String>>> =
std::sync::OnceLock::new();
EMITTED.get_or_init(Default::default)
}
pub(super) fn first_emission_this_process(message: &str) -> bool {
let mut seen = emitted_warnings()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
seen.insert(message.to_string())
}
#[cfg(test)]
pub(super) fn reset_emitted_warnings_for_test() {
emitted_warnings()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clear();
}
impl BackendAssembly {
pub(super) fn new(backends: Vec<BackendConfig>) -> std::result::Result<Self, String> {
validate_backend_names(backends.iter())?;
for b in &backends {
validate_backend_destination(b)?;
}
Ok(Self {
slots: backends.into_iter().map(AssemblySlot::declared).collect(),
pending_probes: Vec::new(),
operator_configured: false,
requested: false,
warnings: Vec::new(),
})
}
fn warn(&mut self, message: String) {
if first_emission_this_process(&message) {
tracing::warn!("{message}");
}
self.warnings.push(message);
}
#[cfg(test)]
pub(super) fn warnings(&self) -> &[String] {
&self.warnings
}
pub(super) fn is_empty(&self) -> bool {
self.slots.is_empty()
}
pub(super) fn operator_configured(&self) -> bool {
self.operator_configured
}
pub(super) fn requested(&self) -> bool {
self.requested
}
pub(super) fn push_fallback(&mut self, backend: BackendConfig) {
self.slots.push(AssemblySlot::declared(backend));
}
fn find(&self, name: &str) -> NameMatch {
let mut hits = self
.slots
.iter()
.enumerate()
.filter(|(_, s)| s.declaration.name == name)
.map(|(i, _)| i);
match (hits.next(), hits.next()) {
(None, _) => NameMatch::Missing,
(Some(i), None) => NameMatch::Unique(i),
(Some(_), Some(_)) => NameMatch::Ambiguous,
}
}
pub(super) fn merge_dir(&mut self, dir: &Path) -> std::result::Result<(), String> {
let Ok(entries) = std::fs::read_dir(dir) else {
return Ok(()); };
let mut paths: Vec<PathBuf> = entries
.flatten()
.map(|e| e.path())
.filter(|p| p.extension().is_some_and(|x| x == "toml"))
.collect();
paths.sort();
for path in paths {
let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else {
continue;
};
let Ok(text) = std::fs::read_to_string(&path) else {
continue;
};
let tag = match disk_record_tag(&text) {
Ok(tag) => tag,
Err(e) => {
self.warn(format!(
"{}: skipping malformed backend file: {e}",
path.display()
));
continue;
}
};
match tag {
Some(RecordTag::ProbeV1) => self.stage_probe(&path, stem, &text),
Some(RecordTag::OperatorV1) => self.merge_operator(&path, stem, &text),
None => {
let backend = match toml::from_str::<BackendConfig>(&text) {
Ok(backend) => backend,
Err(e) => {
self.warn(format!(
"{}: skipping malformed backend file: {e}",
path.display()
));
continue;
}
};
match classify_untagged_dropin(&backend, &text) {
Ok(DropinOwner::Operator) => self.merge_operator(&path, stem, &text),
Ok(DropinOwner::Probe) => self.stage_probe(&path, stem, &text),
Err(reason) => return Err(format!("{}: {reason}", path.display())),
}
}
}
}
Ok(())
}
fn merge_operator(&mut self, path: &Path, stem: &str, text: &str) {
let mut backend = match toml::from_str::<BackendConfig>(text) {
Ok(backend) => backend,
Err(e) => {
self.warn(format!(
"{}: skipping malformed backend file: {e}",
path.display()
));
return;
}
};
backend.name = stem.to_string();
if !backend_has_destination(&backend) {
self.warn(format!(
"{}: skipping backend with neither endpoint nor model_path",
path.display()
));
return;
}
if let Err(reason) = validate_backend_destination(&backend) {
self.warn(format!(
"{}: skipping backend drop-in: {reason}",
path.display()
));
return;
}
self.operator_configured = true;
match self.find(stem) {
NameMatch::Unique(i) => self.slots[i] = AssemblySlot::declared(backend),
NameMatch::Missing => self.slots.push(AssemblySlot::declared(backend)),
NameMatch::Ambiguous => {
self.warn(format!(
"{}: several staged backends share this name — drop-in not merged",
path.display()
));
}
}
}
fn stage_probe(&mut self, path: &Path, stem: &str, text: &str) {
let record = match parse_probe_record(text) {
Ok(record) => record,
Err(reason) => {
self.warn(format!(
"{}: invalid probe record — not overlaid (delete the file to re-probe): {reason}",
path.display()
));
return;
}
};
self.pending_probes.push(PendingProbe {
path: path.to_path_buf(),
stem: stem.to_string(),
observation: record.to_observation(stem),
});
}
fn attach_pending_probes(&mut self) {
for pending in std::mem::take(&mut self.pending_probes) {
let PendingProbe {
path,
stem,
observation,
} = pending;
let slot = match self.find(&stem) {
NameMatch::Unique(i) => &mut self.slots[i],
NameMatch::Missing => {
self.warn(format!(
"{}: probe record names an unconfigured backend — ignored (delete the file)",
path.display()
));
continue;
}
NameMatch::Ambiguous => {
self.warn(format!(
"{}: several staged backends share this name — probe record not attached",
path.display()
));
continue;
}
};
let observed_at = BackendDestination::new(Some(observation.endpoint.clone()), None);
let declared_at = BackendDestination::of(&slot.declaration);
if declared_at != observed_at {
let configured = slot.declaration.endpoint.clone();
let probed = observation.endpoint.clone();
self.warn(format!(
"{}: probe record's destination does not match the configured backend \
(configured={configured}, probed={probed}) — not overlaid",
path.display()
));
continue;
}
slot.observation = Some(observation);
}
}
pub(super) fn apply_request(
&mut self,
over: Option<BackendOverride>,
default_backend: Option<&str>,
) -> std::result::Result<Option<usize>, String> {
self.attach_pending_probes();
let Some(over) = over.filter(|o| !o.is_empty()) else {
return Ok(None);
};
if over.endpoint.as_deref().is_some_and(str::is_empty) {
return Err("--backend-url is empty — give a URL or omit the flag".into());
}
if over.model_path.as_deref().is_some_and(str::is_empty) {
return Err("--backend-model-path is empty — give a path or omit the flag".into());
}
if over.model.as_deref().is_some_and(|m| m.trim().is_empty()) {
return Err(
"--backend-model is empty — give a model or omit the flag (there is \
no implicit clear: the flattened route would serve \
server-decides while the receipt fell back to the stale \
declared model)"
.into(),
);
}
if over.endpoint.is_some() && over.model_path.is_some() {
return Err(
"--backend-url and --backend-model-path are mutually exclusive — a \
backend has ONE destination (an HTTP endpoint, or an embedded \
artifact path)"
.into(),
);
}
if over.endpoint.is_some() && over.kind == Some(BackendKind::Embedded) {
return Err(
"--backend-url with --backend-kind embedded is contradictory — an \
embedded backend has no endpoint; use --backend-model-path"
.into(),
);
}
if over.model_path.is_some() && over.kind.is_some_and(|k| k != BackendKind::Embedded) {
return Err(format!(
"--backend-model-path with --backend-kind {:?} is contradictory — a \
model_path destination is an embedded backend",
over.kind.unwrap()
));
}
if over.model_path.is_some() && over.serving == Some(Serving::Multiplexer) {
return Err(
"--backend-model-path with --backend-serving multiplexer is \
contradictory — an embedded backend serves exactly one artifact \
(instance)"
.into(),
);
}
self.requested = true;
let has_destination = over.endpoint.is_some() || over.model_path.is_some();
if has_destination {
let name = over.name.clone().unwrap_or_else(|| "cli".to_string());
let kept = match self.find(&name) {
NameMatch::Unique(i) => self.slots.swap_remove(i),
NameMatch::Missing => AssemblySlot::declared(BackendConfig {
name: name.clone(),
..Default::default()
}),
NameMatch::Ambiguous => {
return Err(format!(
"--backend-* targets `{name}`, which several backends share — \
rename one"
));
}
};
self.slots = vec![kept];
self.slots[0].request = Some(over);
validate_backend_names(self.slots.iter().map(|s| &s.declaration))?;
return Ok(Some(0));
}
{
let effective: Vec<BackendConfig> =
self.slots.iter().map(AssemblySlot::observed_view).collect();
let idx = match over.name.as_deref() {
Some(n) => match self.find(n) {
NameMatch::Unique(i) => {
if !backend_is_routable(&effective[i]) {
return Err(format!(
"--backend-name `{n}` names a backend with neither an \
endpoint nor a model_path — a field-only --backend-* \
cannot route it; give it a destination \
(--backend-url / --backend-model-path) or fix the \
backend"
));
}
i
}
NameMatch::Missing => {
let configured: Vec<&str> = self
.slots
.iter()
.map(|s| s.declaration.name.as_str())
.collect();
return Err(format!(
"--backend-name `{n}` matches no configured backend \
(configured: {configured:?}) — a field-only --backend-* \
edits an existing backend; add --backend-url to define \
a new one"
));
}
NameMatch::Ambiguous => {
return Err(format!(
"--backend-* targets `{n}`, which several backends share — \
rename one"
));
}
},
None => {
let declarations: Vec<&BackendConfig> = effective.iter().collect();
match select_backend_slot(&declarations, default_backend) {
SlotSelection::Slot(i) => i,
SlotSelection::ExplicitlyUnroutable { name } => {
return Err(format!(
"--backend-* targets `{name}` (named by $NEWT_PROVIDER or \
default_backend), which has neither an endpoint nor a \
model_path — a field-only --backend-* cannot route it; \
give it a destination (--backend-url / \
--backend-model-path) or fix the backend"
));
}
SlotSelection::ExplicitlyUnmatched { name } => {
return Err(format!(
"--backend-* would apply to the selected backend, but \
$NEWT_PROVIDER/default_backend names `{name}`, which \
matches no configured backend (it may name a provider, \
which --backend-* cannot edit) — fix the selector or \
name a backend with --backend-name"
));
}
SlotSelection::None => {
return Err("--backend-* has no backend to apply to — nothing \
configured is routable; name one with --backend-name \
or define one with --backend-url"
.into());
}
}
}
};
if let Some(kind) = over.kind {
let target = &effective[idx];
if kind == BackendKind::Embedded && !target.endpoint.is_empty() {
return Err(format!(
"--backend-kind embedded on `{}` is contradictory — its \
destination is an HTTP endpoint; retarget with \
--backend-model-path or pick an HTTP kind",
target.name
));
}
if kind != BackendKind::Embedded
&& target.endpoint.is_empty()
&& target.model_path.as_deref().is_some_and(|p| !p.is_empty())
{
return Err(format!(
"--backend-kind {kind:?} on `{}` is contradictory — its \
destination is an embedded model_path; retarget with \
--backend-url or keep kind embedded",
target.name
));
}
}
if over.serving == Some(Serving::Multiplexer) {
let target = &effective[idx];
if target.endpoint.is_empty()
&& target.model_path.as_deref().is_some_and(|p| !p.is_empty())
{
return Err(format!(
"--backend-serving multiplexer on `{}` is contradictory — an \
embedded (model_path) backend serves exactly one artifact \
(instance); retarget with --backend-url for a multiplexer",
target.name
));
}
}
if over.name.is_none() {
let mut post: Vec<BackendConfig> = effective.clone();
over.overlay(&mut post[idx]);
let post_refs: Vec<&BackendConfig> = post.iter().collect();
match select_backend_slot(&post_refs, default_backend) {
SlotSelection::Slot(i) if i == idx => {}
_ => {
return Err(format!(
"--backend-* would edit `{}` (the currently selected \
backend), but the edit changes which backend the shared \
precedence selects — name the target explicitly with \
--backend-name",
self.slots[idx].declaration.name
));
}
}
}
self.slots[idx].request = Some(over);
validate_backend_names(self.slots.iter().map(|s| &s.declaration))?;
Ok(Some(idx))
}
}
pub(super) fn finish(mut self) -> (Vec<BackendConfig>, Vec<BackendResolutionReceipt>) {
self.attach_pending_probes();
let mut backends = Vec::with_capacity(self.slots.len());
let mut receipts = Vec::with_capacity(self.slots.len());
for slot in self.slots {
let declaration = DeclaredBackend::of(&slot.declaration);
let request = slot.request.as_ref().map(BackendRequest::from_override);
let destination = request
.as_ref()
.map(|r| r.destination_over(&declaration.destination))
.unwrap_or_else(|| declaration.destination.clone());
let observation = slot
.observation
.filter(|_| destination == declaration.destination);
let mut backend = slot.declaration;
if let Some(obs) = &observation {
overlay_observation(&mut backend, obs);
}
if let Some(over) = &slot.request {
over.overlay(&mut backend);
let exclusive = over.endpoint.is_some() || over.model_path.is_some();
if exclusive && backend.tiers.is_empty() {
backend.tiers = vec![Tier::Fast, Tier::Standard, Tier::Complex, Tier::Review];
}
}
normalize_destination_kind(&mut backend);
let binding = match &request {
Some(req) if req.card.is_some() => crate::model_card::CardBindingSeed {
card: req.card.clone(),
bound_model: req.model.clone().or_else(|| declaration.model.clone()),
bound_destination: destination.clone(),
},
_ => crate::model_card::CardBindingSeed {
card: declaration.card.clone(),
bound_model: declaration.model.clone(),
bound_destination: declaration.destination.clone(),
},
};
receipts.push(BackendResolutionReceipt {
declaration,
request,
observation,
binding,
});
backends.push(backend);
}
(backends, receipts)
}
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct BackendOverride {
pub name: Option<String>,
pub endpoint: Option<String>,
pub model: Option<String>,
pub model_path: Option<String>,
pub tiers: Option<Vec<Tier>>,
pub kind: Option<BackendKind>,
pub api: Option<OpenAiApi>,
pub api_key_env: Option<String>,
pub api_key_file: Option<String>,
pub serving: Option<Serving>,
pub engine: Option<Engine>,
pub host: Option<String>,
pub coexist: Option<bool>,
pub ram_gib: Option<f64>,
pub card: Option<String>,
}
impl BackendOverride {
pub fn is_empty(&self) -> bool {
*self == Self::default()
}
pub fn apply(&self, cfg: &mut Config) {
if let Err(e) = self.try_apply(cfg) {
tracing::warn!(error = %e, "--backend-* override not applied");
}
}
pub fn try_apply(&self, cfg: &mut Config) -> std::result::Result<(), String> {
if self.is_empty() {
return Ok(());
}
let original = cfg.backends.clone();
let mut assembly = match BackendAssembly::new(std::mem::take(&mut cfg.backends)) {
Ok(assembly) => assembly,
Err(e) => {
cfg.backends = original;
return Err(e);
}
};
let default_backend = cfg.default_backend.clone();
let applied = assembly.apply_request(Some(self.clone()), default_backend.as_deref());
let (backends, _receipts) = assembly.finish();
match applied {
Ok(target) => {
cfg.backends = backends;
cfg.backend_fallback = false;
pin_requested_selection(cfg, Some(self), target);
Ok(())
}
Err(e) => {
cfg.backends = original;
Err(e)
}
}
}
fn overlay(&self, backend: &mut BackendConfig) {
if self.endpoint.is_some() || self.model_path.is_some() {
backend.endpoint = String::new();
backend.model_path = None;
}
if let Some(v) = &self.endpoint {
backend.endpoint = v.clone();
}
if let Some(v) = &self.model {
backend.model = Some(v.clone());
}
if let Some(v) = &self.model_path {
backend.model_path = Some(v.clone());
}
if let Some(v) = &self.tiers {
backend.tiers = v.clone();
}
if let Some(v) = self.kind {
backend.kind = Some(v);
}
if let Some(v) = self.api {
backend.api = Some(v);
}
if let Some(v) = &self.api_key_env {
backend.api_key_env = Some(v.clone());
}
if let Some(v) = &self.api_key_file {
backend.api_key_file = Some(v.clone());
}
if let Some(v) = self.serving {
backend.serving = Some(v);
}
if let Some(v) = self.engine {
backend.engine = Some(v);
}
if let Some(v) = &self.host {
backend.host = Some(v.clone());
}
if let Some(v) = self.coexist {
backend.coexist = Some(v);
}
if let Some(v) = self.ram_gib {
backend.ram_gib = Some(v);
}
if let Some(v) = &self.card {
backend.card = Some(v.clone());
}
}
}
static CLI_BACKEND_OVERRIDE: std::sync::Mutex<Option<BackendOverride>> =
std::sync::Mutex::new(None);
pub fn set_cli_backend_override(over: BackendOverride) {
if let Ok(mut slot) = CLI_BACKEND_OVERRIDE.lock() {
*slot = Some(over);
}
}
pub(super) fn cli_backend_override() -> Option<BackendOverride> {
CLI_BACKEND_OVERRIDE.lock().ok().and_then(|s| s.clone())
}