1use crate::{
2 adapters::{BackendAdapter, BackendError, ServeRequest, model_id},
3 command::CommandRunner,
4 http::{Request, Transport, read_json},
5 identity::{Listener, ProcessIdentity, ProcessProbe},
6 process_control::{
7 ProcessControl, SpawnSpec, listener, minimal_env, same_process, stop_owned,
8 wait_owned_with_timeout,
9 },
10 state::{RuntimeState, ServerState, secure_read},
11};
12use serde::Deserialize;
13use serde_json::{Value, json};
14use std::{
15 path::{Path, PathBuf},
16 time::Duration,
17};
18use tokio_util::sync::CancellationToken;
19
20async fn request(
21 http: &dyn Transport,
22 endpoint: &str,
23 path: &str,
24 body: Option<Value>,
25 token: Option<&str>,
26 cancel: &CancellationToken,
27) -> Result<Value, BackendError> {
28 let request = Request::new(endpoint, path, body, token.map(str::to_owned))
29 .map_err(|error| BackendError(error.to_string()))?;
30 tokio::time::timeout(
31 Duration::from_secs(5),
32 read_json(http, request, cancel, 2 * 1024 * 1024),
33 )
34 .await
35 .map_err(|_| BackendError("runtime request timed out".into()))?
36 .map_err(|error| BackendError(error.to_string()))
37}
38fn state(
39 name: &str,
40 request: &ServeRequest,
41 observed: Listener,
42 owned: bool,
43 token: Option<String>,
44) -> Result<ServerState, BackendError> {
45 let mut value = json!({"backend":name,"modelId":request.model_id,"endpoint":request.endpoint,"port":observed.port,"ownedByUs":owned,"pid":observed.identity.pid,"processExecutable":observed.identity.executable,"processStartedAt":observed.identity.started});
46 if name == "lmstudio" {
47 value["modelPath"] = json!(
48 request
49 .model_path
50 .as_ref()
51 .ok_or_else(|| BackendError("missing delegated path".into()))?
52 .to_string_lossy()
53 );
54 }
55 if let Some(token) = token {
56 value["authToken"] = json!(token);
57 }
58 RuntimeState::parse(&json!({"schemaVersion":2,"active":value}).to_string())
59 .map_err(|error| BackendError(error.to_string()))?
60 .active
61 .ok_or_else(|| BackendError("missing runtime state".into()))
62}
63fn ids(value: Value) -> Result<Vec<String>, BackendError> {
64 #[derive(Deserialize)]
65 struct Model {
66 id: String,
67 }
68 #[derive(Deserialize)]
69 struct Models {
70 data: Vec<Model>,
71 }
72 let models: Models =
73 serde_json::from_value(value).map_err(|_| BackendError("invalid runtime models".into()))?;
74 if models.data.len() > 10000
75 || models
76 .data
77 .iter()
78 .any(|model| model.id.is_empty() || model.id.len() > 4096)
79 {
80 return Err(BackendError("invalid runtime model identifiers".into()));
81 }
82 Ok(models.data.into_iter().map(|model| model.id).collect())
83}
84fn validate(request: &ServeRequest) -> Result<url::Url, BackendError> {
85 model_id(&request.model_id)?;
86 if request.context.is_some() {
87 return Err(BackendError(
88 "explicit context currently requires Ollama or llama.cpp".into(),
89 ));
90 }
91 crate::state::loopback(&request.endpoint).map_err(|error| BackendError(error.to_string()))
92}
93pub struct LmStudioAdapter<'runtime> {
94 pub binary: PathBuf,
95 pub trusted_executables: Vec<PathBuf>,
96 pub http: &'runtime dyn Transport,
97 pub probe: &'runtime dyn ProcessProbe,
98 pub commands: &'runtime dyn CommandRunner,
99 pub token: Option<String>,
100}
101pub fn trusted_executable(trusted: &[PathBuf], observed: &str, windows: bool) -> bool {
103 let normalize = |path: &str| {
104 let path = path.replace('/', "\\");
105 path.strip_prefix("\\\\?\\")
106 .map(str::to_owned)
107 .unwrap_or(path)
108 .to_lowercase()
109 };
110 trusted.iter().any(|path| {
111 let Some(path) = path.to_str() else {
112 return false;
113 };
114 if windows {
115 let path = normalize(path);
116 path.as_bytes().get(1) == Some(&b':') && path == normalize(observed)
117 } else {
118 path.starts_with('/') && path == observed
119 }
120 })
121}
122impl LmStudioAdapter<'_> {
123 async fn checked(
124 &self,
125 input: &ServeRequest,
126 cancel: &CancellationToken,
127 ) -> Result<Listener, BackendError> {
128 let url = validate(input)?;
129 let path = input
130 .model_path
131 .as_ref()
132 .and_then(|path| path.to_str())
133 .filter(|path| {
134 !path.is_empty() && path.len() <= 4096 && !path.chars().any(char::is_control)
135 })
136 .ok_or_else(|| BackendError("exact delegated path required".into()))?;
137 let before = listener(self.probe, &input.endpoint, cancel).await?;
138 if !trusted_executable(
139 &self.trusted_executables,
140 &before.identity.executable,
141 cfg!(windows),
142 ) {
143 return Err(BackendError("untrusted LM Studio executable".into()));
144 }
145 let greeting = request(
146 self.http,
147 &input.endpoint,
148 "/lmstudio-greeting",
149 None,
150 self.token.as_deref(),
151 cancel,
152 )
153 .await?;
154 if greeting.get("lmstudio") != Some(&Value::Bool(true)) {
155 return Err(BackendError("listener is not LM Studio".into()));
156 }
157 let status = self
158 .commands
159 .run(
160 &self.binary,
161 &[
162 "server".into(),
163 "status".into(),
164 "--json".into(),
165 "--quiet".into(),
166 ],
167 cancel,
168 Duration::from_secs(5),
169 )
170 .await?;
171 #[derive(Deserialize)]
172 struct Status {
173 running: bool,
174 port: u16,
175 }
176 let status: Status = serde_json::from_str(&status)
177 .map_err(|_| BackendError("invalid LM Studio server status".into()))?;
178 if !status.running || Some(status.port) != url.port_or_known_default() {
179 return Err(BackendError("LM Studio server port mismatch".into()));
180 }
181 if !ids(request(
182 self.http,
183 &input.endpoint,
184 "/v1/models",
185 None,
186 self.token.as_deref(),
187 cancel,
188 )
189 .await?)?
190 .iter()
191 .any(|id| id == &input.model_id)
192 {
193 return Err(BackendError("LM Studio model not loaded".into()));
194 }
195 let loaded = self
196 .commands
197 .run(
198 &self.binary,
199 &["ps".into(), "--json".into(), "--quiet".into()],
200 cancel,
201 Duration::from_secs(5),
202 )
203 .await?;
204 #[derive(Deserialize)]
205 struct Loaded {
206 identifier: String,
207 path: String,
208 }
209 let loaded: Vec<Loaded> = serde_json::from_str(&loaded)
210 .map_err(|_| BackendError("invalid LM Studio loaded models".into()))?;
211 if loaded.len() > 1024
212 || loaded.iter().any(|entry| {
213 entry.identifier.is_empty()
214 || entry.identifier.len() > 1024
215 || entry.path.is_empty()
216 || entry.path.len() > 4096
217 })
218 || loaded
219 .iter()
220 .filter(|entry| {
221 entry.identifier == input.model_id
222 && entry.path.replace('\\', "/") == path.replace('\\', "/")
223 })
224 .count()
225 != 1
226 {
227 return Err(BackendError(
228 "LM Studio exact delegated path not loaded".into(),
229 ));
230 }
231 let after = listener(self.probe, &input.endpoint, cancel).await?;
232 if !same_process(&before.identity, &after.identity) {
233 return Err(BackendError("LM Studio listener changed".into()));
234 }
235 Ok(after)
236 }
237}
238#[async_trait::async_trait]
239impl BackendAdapter for LmStudioAdapter<'_> {
240 async fn chat(
241 &self,
242 handle: &ServerState,
243 input: &crate::ollama_inference::ChatInput,
244 cancel: &CancellationToken,
245 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
246 crate::openai::OpenAiInference {
247 http: self.http,
248 probe: self.probe,
249 adapter: self,
250 token: self.token.as_deref(),
251 }
252 .chat(handle, input, cancel)
253 .await
254 }
255 async fn embed(
256 &self,
257 handle: &ServerState,
258 model: &str,
259 inputs: &[String],
260 cancel: &CancellationToken,
261 ) -> Result<crate::ollama_inference::EmbedResult, BackendError> {
262 crate::openai::OpenAiInference {
263 http: self.http,
264 probe: self.probe,
265 adapter: self,
266 token: self.token.as_deref(),
267 }
268 .embed(handle, model, inputs, cancel)
269 .await
270 }
271 fn name(&self) -> &'static str {
272 "lmstudio"
273 }
274 fn trusts(&self, identity: &ProcessIdentity) -> bool {
275 trusted_executable(
276 &self.trusted_executables,
277 &identity.executable,
278 cfg!(windows),
279 )
280 }
281 async fn serve(
282 &self,
283 input: &ServeRequest,
284 cancel: &CancellationToken,
285 ) -> Result<ServerState, BackendError> {
286 state(
287 self.name(),
288 input,
289 self.checked(input, cancel).await?,
290 false,
291 None,
292 )
293 }
294 async fn ready(
295 &self,
296 input: &ServeRequest,
297 cancel: &CancellationToken,
298 ) -> Result<(), BackendError> {
299 self.checked(input, cancel).await.map(|_| ())
300 }
301 async fn stop(
302 &self,
303 handle: &ServerState,
304 _cancel: &CancellationToken,
305 ) -> Result<(), BackendError> {
306 handle
307 .validate()
308 .map_err(|error| BackendError(error.to_string()))?;
309 if handle.backend != "lmstudio" || handle.owned_by_us {
310 return Err(BackendError("invalid LM Studio handle".into()));
311 }
312 Ok(())
313 }
314}
315
316pub struct MlxAdapter<'runtime> {
317 pub binary: PathBuf,
318 pub http: &'runtime dyn Transport,
319 pub probe: &'runtime dyn ProcessProbe,
320 pub control: &'runtime dyn ProcessControl,
321 pub commands: &'runtime dyn CommandRunner,
322 pub token: Option<String>,
323}
324pub fn mlx_process_executable(binary: &Path) -> PathBuf {
326 fn name(path: Option<&Path>) -> Option<&str> {
327 path.and_then(Path::file_name)
328 .and_then(|name| name.to_str())
329 }
330 let bin = binary.parent();
331 let version = bin.and_then(Path::parent);
332 let versions = version.and_then(Path::parent);
333 let framework = versions.and_then(Path::parent);
334 let interpreter = name(Some(binary)).is_some_and(|name| {
335 name.strip_prefix("python").is_some_and(|rest| {
336 rest.chars()
337 .all(|character| character.is_ascii_digit() || character == '.')
338 })
339 });
340 match version {
341 Some(version)
342 if interpreter
343 && name(bin) == Some("bin")
344 && name(versions) == Some("Versions")
345 && name(framework) == Some("Python.framework") =>
346 {
347 version.join("Resources/Python.app/Contents/MacOS/Python")
348 }
349 _ => binary.to_path_buf(),
350 }
351}
352pub fn validate_mlx_directory(root: &Path) -> Result<(), BackendError> {
353 if !root.is_absolute()
354 || !root.is_dir()
355 || std::fs::symlink_metadata(root)
356 .map_err(|_| BackendError("model root unavailable".into()))?
357 .file_type()
358 .is_symlink()
359 {
360 return Err(BackendError(
361 "verified absolute MLX directory required".into(),
362 ));
363 }
364 fn scan(path: &Path, count: &mut usize, weights: &mut bool) -> Result<(), BackendError> {
365 for entry in std::fs::read_dir(path)
366 .map_err(|_| BackendError("cannot inspect model directory".into()))?
367 {
368 let entry = entry.map_err(|_| BackendError("cannot inspect model entry".into()))?;
369 *count += 1;
370 if *count > 1024 {
371 return Err(BackendError("model directory exceeds entry limit".into()));
372 }
373 let kind = entry
374 .file_type()
375 .map_err(|_| BackendError("cannot inspect model entry type".into()))?;
376 if kind.is_dir() {
377 scan(&entry.path(), count, weights)?;
378 } else if kind.is_file() {
379 let name = entry.file_name().to_string_lossy().to_ascii_lowercase();
380 if ["py", "pyc", "pyo", "so", "dylib", "dll", "bundle"]
381 .iter()
382 .any(|extension| name.ends_with(&format!(".{extension}")))
383 {
384 return Err(BackendError("executable MLX artifact forbidden".into()));
385 }
386 *weights |= name.ends_with(".safetensors");
387 } else {
388 return Err(BackendError(
389 "symlink or special MLX artifact forbidden".into(),
390 ));
391 }
392 }
393 Ok(())
394 }
395 let mut count = 0;
396 let mut weights = false;
397 scan(root, &mut count, &mut weights)?;
398 if !weights {
399 return Err(BackendError("MLX weights absent".into()));
400 }
401 for file in ["config.json", "tokenizer_config.json"] {
402 let raw = secure_read(&root.join(file), 1048576, false)
403 .map_err(|_| BackendError("MLX config unavailable".into()))?;
404 let value: Value =
405 serde_json::from_str(&raw).map_err(|_| BackendError("invalid MLX config".into()))?;
406 if !value.is_object()
407 || ["auto_map", "model_file"]
408 .iter()
409 .any(|key| value.get(key).is_some_and(|value| !value.is_null()))
410 || value.get("trust_remote_code") == Some(&Value::Bool(true))
411 {
412 return Err(BackendError("MLX remote code config forbidden".into()));
413 }
414 }
415 Ok(())
416}
417impl MlxAdapter<'_> {
418 async fn ready_once(
419 &self,
420 input: &ServeRequest,
421 cancel: &CancellationToken,
422 ) -> Result<(), BackendError> {
423 let token = self
424 .token
425 .as_deref()
426 .filter(|value| value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()))
427 .ok_or_else(|| BackendError("MLX session token required".into()))?;
428 let health = request(
429 self.http,
430 &input.endpoint,
431 "/health",
432 None,
433 Some(token),
434 cancel,
435 )
436 .await?;
437 if health.get("status").and_then(Value::as_str) != Some("ok") {
438 return Err(BackendError("MLX health not ready".into()));
439 }
440 let path = input
441 .model_path
442 .as_ref()
443 .and_then(|path| path.to_str())
444 .ok_or_else(|| BackendError("MLX model path required".into()))?;
445 if !ids(request(
446 self.http,
447 &input.endpoint,
448 "/v1/models",
449 None,
450 Some(token),
451 cancel,
452 )
453 .await?)?
454 .iter()
455 .any(|id| id == path)
456 {
457 return Err(BackendError("MLX loaded path mismatch".into()));
458 }
459 let completion=request(self.http,&input.endpoint,"/v1/chat/completions",Some(json!({"model":"default_model","messages":[{"role":"user","content":"Reply briefly."}],"max_tokens":1,"stream":false})),Some(token),cancel).await?;
460 if !completion
461 .get("choices")
462 .and_then(Value::as_array)
463 .and_then(|choices| choices.first())
464 .and_then(|choice| choice.get("message"))
465 .and_then(|message| message.get("content"))
466 .is_some_and(Value::is_string)
467 {
468 return Err(BackendError("MLX identity completion failed".into()));
469 }
470 Ok(())
471 }
472}
473#[async_trait::async_trait]
474impl BackendAdapter for MlxAdapter<'_> {
475 async fn chat(
476 &self,
477 handle: &ServerState,
478 input: &crate::ollama_inference::ChatInput,
479 cancel: &CancellationToken,
480 ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
481 crate::openai::OpenAiInference {
482 http: self.http,
483 probe: self.probe,
484 adapter: self,
485 token: None,
486 }
487 .chat(handle, input, cancel)
488 .await
489 }
490 fn name(&self) -> &'static str {
491 "mlx"
492 }
493 fn trusts(&self, identity: &ProcessIdentity) -> bool {
494 self.binary.to_str() == Some(identity.executable.as_str())
495 || mlx_process_executable(&self.binary).to_str() == Some(identity.executable.as_str())
496 }
497 async fn ready_handle(
498 &self,
499 input: &ServeRequest,
500 handle: &ServerState,
501 cancel: &CancellationToken,
502 ) -> Result<(), BackendError> {
503 let session = MlxAdapter {
504 binary: self.binary.clone(),
505 http: self.http,
506 probe: self.probe,
507 control: self.control,
508 commands: self.commands,
509 token: handle.auth_token.clone(),
510 };
511 session.ready(input, cancel).await
512 }
513 async fn serve(
514 &self,
515 input: &ServeRequest,
516 cancel: &CancellationToken,
517 ) -> Result<ServerState, BackendError> {
518 if !cfg!(all(target_os = "macos", target_arch = "aarch64")) {
519 return Err(BackendError("MLX requires Apple Silicon".into()));
520 }
521 let url = validate(input)?;
522 if cancel.is_cancelled() {
523 return Err(BackendError("cancelled".into()));
524 }
525 if self.control.occupied(&input.endpoint).await? {
526 return Err(BackendError(
527 "MLX never attaches to occupied endpoints".into(),
528 ));
529 }
530 let path = input
531 .model_path
532 .as_ref()
533 .ok_or_else(|| BackendError("MLX model path required".into()))?;
534 validate_mlx_directory(path)?;
535 let version=self.commands.run(&self.binary,&["-I".into(),"-c".into(),"import importlib.metadata as m; from mlx_lm import server as s; assert hasattr(s, '_run_http_server') and hasattr(s, 'APIHandler'); print(m.version('mlx-lm'))".into()],cancel,Duration::from_secs(5)).await?;
536 if version.trim() != "0.31.3" {
537 return Err(BackendError("MLX requires audited mlx-lm 0.31.3".into()));
538 }
539 let token = format!(
540 "{}{}",
541 uuid::Uuid::new_v4().simple(),
542 uuid::Uuid::new_v4().simple()
543 );
544 let mut env = minimal_env();
545 env.insert("LLMUP_MLX_AUTH_TOKEN".into(), token.clone());
546 let args = vec![
547 "-I".into(),
548 "-c".into(),
549 MLX_WRAPPER.into(),
550 "mlx_lm.server".into(),
551 "--model".into(),
552 path.to_string_lossy().into_owned(),
553 "--host".into(),
554 url.host_str()
555 .ok_or_else(|| BackendError("missing MLX host".into()))?
556 .trim_matches(['[', ']'])
557 .into(),
558 "--port".into(),
559 url.port_or_known_default()
560 .ok_or_else(|| BackendError("missing MLX port".into()))?
561 .to_string(),
562 "--allowed-origins".into(),
563 String::new(),
564 "--log-level".into(),
565 "ERROR".into(),
566 ];
567 if cancel.is_cancelled() {
568 return Err(BackendError("cancelled".into()));
569 }
570 let child = self
571 .control
572 .spawn(&SpawnSpec {
573 binary: self.binary.clone(),
574 args,
575 env,
576 })
577 .await?;
578 let session = MlxAdapter {
579 binary: self.binary.clone(),
580 http: self.http,
581 probe: self.probe,
582 control: self.control,
583 commands: self.commands,
584 token: Some(token.clone()),
585 };
586 let observed = wait_owned_with_timeout(
587 child,
588 &input.endpoint,
589 mlx_process_executable(&self.binary)
590 .to_str()
591 .ok_or_else(|| BackendError("invalid interpreter path".into()))?,
592 self.probe,
593 cancel,
594 Duration::from_secs(300),
595 || async { session.ready(input, cancel).await.map_err(|error| error.0) },
596 )
597 .await?;
598 state(self.name(), input, observed, true, Some(token))
599 }
600 async fn ready(
601 &self,
602 input: &ServeRequest,
603 cancel: &CancellationToken,
604 ) -> Result<(), BackendError> {
605 validate(input)?;
606 let operation = async {
607 for _ in 0..150 {
608 if self.ready_once(input, cancel).await.is_ok() {
609 return Ok(());
610 }
611 tokio::time::sleep(Duration::from_millis(250)).await;
612 }
613 Err(BackendError("MLX readiness exhausted".into()))
614 };
615 tokio::select! {biased;_=cancel.cancelled()=>Err(BackendError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(300),operation)=>result.map_err(|_|BackendError("MLX readiness timed out".into()))?}
616 }
617 async fn stop(
618 &self,
619 handle: &ServerState,
620 cancel: &CancellationToken,
621 ) -> Result<(), BackendError> {
622 handle
623 .validate()
624 .map_err(|error| BackendError(error.to_string()))?;
625 if handle.backend != "mlx" || !handle.owned_by_us {
626 return Err(BackendError("invalid owned MLX handle".into()));
627 }
628 let expected = ProcessIdentity {
629 pid: handle
630 .pid
631 .ok_or_else(|| BackendError("missing MLX PID".into()))?,
632 process: String::new(),
633 executable: handle
634 .process_executable
635 .clone()
636 .ok_or_else(|| BackendError("missing MLX executable".into()))?,
637 started: handle
638 .process_started_at
639 .clone()
640 .ok_or_else(|| BackendError("missing MLX start identity".into()))?,
641 };
642 stop_owned(
643 &handle.endpoint,
644 &expected,
645 self.probe,
646 self.control,
647 cancel,
648 )
649 .await
650 .map_err(BackendError)
651 }
652}
653const MLX_WRAPPER: &str = r#"
654import hmac, io, json, os, sys
655from mlx_lm import server as _server
656_original_run = _server._run_http_server
657class GuardedHandler(_server.APIHandler):
658 def _reject(self, status):
659 self.send_response(status)
660 self.send_header("Content-Length", "0")
661 self.end_headers()
662 def do_OPTIONS(self):
663 self._reject(403)
664 def _authenticated(self):
665 return hmac.compare_digest(self.headers.get("Authorization", ""), "Bearer " + os.environ["LLMUP_MLX_AUTH_TOKEN"])
666 def do_GET(self):
667 if not self._authenticated():
668 self._reject(401)
669 return
670 super().do_GET()
671 def do_POST(self):
672 if self.headers.get("Origin") not in (None, "http://127.0.0.1", "http://localhost"):
673 self._reject(403)
674 return
675 if not self._authenticated():
676 self._reject(401)
677 return
678 if self.headers.get("Content-Type", "").split(";", 1)[0].strip().lower() != "application/json":
679 self._reject(415)
680 return
681 try:
682 length = int(self.headers.get("Content-Length", ""))
683 except ValueError:
684 self._reject(411)
685 return
686 if length < 0 or length > 4194304:
687 self._reject(413)
688 return
689 raw = self.rfile.read(length)
690 try:
691 body = json.loads(raw.decode("utf-8"))
692 except (UnicodeDecodeError, json.JSONDecodeError):
693 self._reject(400)
694 return
695 if not isinstance(body, dict) or body.get("model", "default_model") != "default_model":
696 self._reject(400)
697 return
698 maximum = body.get("max_completion_tokens", body.get("max_tokens", 512))
699 if not isinstance(maximum, int) or isinstance(maximum, bool) or not 0 <= maximum <= 4096:
700 self._reject(400)
701 return
702 self.rfile = io.BytesIO(raw)
703 super().do_POST()
704def _guarded_run(host, port, response_generator, server_class=_server.ThreadingHTTPServer, handler_class=GuardedHandler):
705 return _original_run(host, port, response_generator, server_class, GuardedHandler)
706_server._run_http_server = _guarded_run
707sys.argv.pop(1)
708_server.main()
709"#;