Skip to main content

rigspark_runtime/
special_adapters.rs

1use crate::{
2    adapters::{BackendAdapter, BackendError, ServeRequest, model_id},
3    command::CommandRunner,
4    http::{Request, Transport, read_json},
5    identity::{Listener, ProcessIdentity, ProcessProbe},
6    process_control::{
7        ProcessControl, SpawnSpec, listener, minimal_env, same_process, stop_owned,
8        wait_owned_with_timeout,
9    },
10    state::{RuntimeState, ServerState, secure_read},
11};
12use serde::Deserialize;
13use serde_json::{Value, json};
14use std::{
15    path::{Path, PathBuf},
16    time::Duration,
17};
18use tokio_util::sync::CancellationToken;
19
20async fn request(
21    http: &dyn Transport,
22    endpoint: &str,
23    path: &str,
24    body: Option<Value>,
25    token: Option<&str>,
26    cancel: &CancellationToken,
27) -> Result<Value, BackendError> {
28    let request = Request::new(endpoint, path, body, token.map(str::to_owned))
29        .map_err(|error| BackendError(error.to_string()))?;
30    tokio::time::timeout(
31        Duration::from_secs(5),
32        read_json(http, request, cancel, 2 * 1024 * 1024),
33    )
34    .await
35    .map_err(|_| BackendError("runtime request timed out".into()))?
36    .map_err(|error| BackendError(error.to_string()))
37}
38fn state(
39    name: &str,
40    request: &ServeRequest,
41    observed: Listener,
42    owned: bool,
43    token: Option<String>,
44) -> Result<ServerState, BackendError> {
45    let mut value = json!({"backend":name,"modelId":request.model_id,"endpoint":request.endpoint,"port":observed.port,"ownedByUs":owned,"pid":observed.identity.pid,"processExecutable":observed.identity.executable,"processStartedAt":observed.identity.started});
46    if name == "lmstudio" {
47        value["modelPath"] = json!(
48            request
49                .model_path
50                .as_ref()
51                .ok_or_else(|| BackendError("missing delegated path".into()))?
52                .to_string_lossy()
53        );
54    }
55    if let Some(token) = token {
56        value["authToken"] = json!(token);
57    }
58    RuntimeState::parse(&json!({"schemaVersion":2,"active":value}).to_string())
59        .map_err(|error| BackendError(error.to_string()))?
60        .active
61        .ok_or_else(|| BackendError("missing runtime state".into()))
62}
63fn ids(value: Value) -> Result<Vec<String>, BackendError> {
64    #[derive(Deserialize)]
65    struct Model {
66        id: String,
67    }
68    #[derive(Deserialize)]
69    struct Models {
70        data: Vec<Model>,
71    }
72    let models: Models =
73        serde_json::from_value(value).map_err(|_| BackendError("invalid runtime models".into()))?;
74    if models.data.len() > 10000
75        || models
76            .data
77            .iter()
78            .any(|model| model.id.is_empty() || model.id.len() > 4096)
79    {
80        return Err(BackendError("invalid runtime model identifiers".into()));
81    }
82    Ok(models.data.into_iter().map(|model| model.id).collect())
83}
84fn validate(request: &ServeRequest) -> Result<url::Url, BackendError> {
85    model_id(&request.model_id)?;
86    if request.context.is_some() {
87        return Err(BackendError(
88            "explicit context currently requires Ollama or llama.cpp".into(),
89        ));
90    }
91    crate::state::loopback(&request.endpoint).map_err(|error| BackendError(error.to_string()))
92}
93pub struct LmStudioAdapter<'runtime> {
94    pub binary: PathBuf,
95    pub trusted_executables: Vec<PathBuf>,
96    pub http: &'runtime dyn Transport,
97    pub probe: &'runtime dyn ProcessProbe,
98    pub commands: &'runtime dyn CommandRunner,
99    pub token: Option<String>,
100}
101/// Windows reports `C:\...` while canonical paths are `\\?\C:\...`, and file names are case-insensitive.
102pub fn trusted_executable(trusted: &[PathBuf], observed: &str, windows: bool) -> bool {
103    let normalize = |path: &str| {
104        let path = path.replace('/', "\\");
105        path.strip_prefix("\\\\?\\")
106            .map(str::to_owned)
107            .unwrap_or(path)
108            .to_lowercase()
109    };
110    trusted.iter().any(|path| {
111        let Some(path) = path.to_str() else {
112            return false;
113        };
114        if windows {
115            let path = normalize(path);
116            path.as_bytes().get(1) == Some(&b':') && path == normalize(observed)
117        } else {
118            path.starts_with('/') && path == observed
119        }
120    })
121}
122impl LmStudioAdapter<'_> {
123    async fn checked(
124        &self,
125        input: &ServeRequest,
126        cancel: &CancellationToken,
127    ) -> Result<Listener, BackendError> {
128        let url = validate(input)?;
129        let path = input
130            .model_path
131            .as_ref()
132            .and_then(|path| path.to_str())
133            .filter(|path| {
134                !path.is_empty() && path.len() <= 4096 && !path.chars().any(char::is_control)
135            })
136            .ok_or_else(|| BackendError("exact delegated path required".into()))?;
137        let before = listener(self.probe, &input.endpoint, cancel).await?;
138        if !trusted_executable(
139            &self.trusted_executables,
140            &before.identity.executable,
141            cfg!(windows),
142        ) {
143            return Err(BackendError("untrusted LM Studio executable".into()));
144        }
145        let greeting = request(
146            self.http,
147            &input.endpoint,
148            "/lmstudio-greeting",
149            None,
150            self.token.as_deref(),
151            cancel,
152        )
153        .await?;
154        if greeting.get("lmstudio") != Some(&Value::Bool(true)) {
155            return Err(BackendError("listener is not LM Studio".into()));
156        }
157        let status = self
158            .commands
159            .run(
160                &self.binary,
161                &[
162                    "server".into(),
163                    "status".into(),
164                    "--json".into(),
165                    "--quiet".into(),
166                ],
167                cancel,
168                Duration::from_secs(5),
169            )
170            .await?;
171        #[derive(Deserialize)]
172        struct Status {
173            running: bool,
174            port: u16,
175        }
176        let status: Status = serde_json::from_str(&status)
177            .map_err(|_| BackendError("invalid LM Studio server status".into()))?;
178        if !status.running || Some(status.port) != url.port_or_known_default() {
179            return Err(BackendError("LM Studio server port mismatch".into()));
180        }
181        if !ids(request(
182            self.http,
183            &input.endpoint,
184            "/v1/models",
185            None,
186            self.token.as_deref(),
187            cancel,
188        )
189        .await?)?
190        .iter()
191        .any(|id| id == &input.model_id)
192        {
193            return Err(BackendError("LM Studio model not loaded".into()));
194        }
195        let loaded = self
196            .commands
197            .run(
198                &self.binary,
199                &["ps".into(), "--json".into(), "--quiet".into()],
200                cancel,
201                Duration::from_secs(5),
202            )
203            .await?;
204        #[derive(Deserialize)]
205        struct Loaded {
206            identifier: String,
207            path: String,
208        }
209        let loaded: Vec<Loaded> = serde_json::from_str(&loaded)
210            .map_err(|_| BackendError("invalid LM Studio loaded models".into()))?;
211        if loaded.len() > 1024
212            || loaded.iter().any(|entry| {
213                entry.identifier.is_empty()
214                    || entry.identifier.len() > 1024
215                    || entry.path.is_empty()
216                    || entry.path.len() > 4096
217            })
218            || loaded
219                .iter()
220                .filter(|entry| {
221                    entry.identifier == input.model_id
222                        && entry.path.replace('\\', "/") == path.replace('\\', "/")
223                })
224                .count()
225                != 1
226        {
227            return Err(BackendError(
228                "LM Studio exact delegated path not loaded".into(),
229            ));
230        }
231        let after = listener(self.probe, &input.endpoint, cancel).await?;
232        if !same_process(&before.identity, &after.identity) {
233            return Err(BackendError("LM Studio listener changed".into()));
234        }
235        Ok(after)
236    }
237}
238#[async_trait::async_trait]
239impl BackendAdapter for LmStudioAdapter<'_> {
240    async fn chat(
241        &self,
242        handle: &ServerState,
243        input: &crate::ollama_inference::ChatInput,
244        cancel: &CancellationToken,
245    ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
246        crate::openai::OpenAiInference {
247            http: self.http,
248            probe: self.probe,
249            adapter: self,
250            token: self.token.as_deref(),
251        }
252        .chat(handle, input, cancel)
253        .await
254    }
255    async fn embed(
256        &self,
257        handle: &ServerState,
258        model: &str,
259        inputs: &[String],
260        cancel: &CancellationToken,
261    ) -> Result<crate::ollama_inference::EmbedResult, BackendError> {
262        crate::openai::OpenAiInference {
263            http: self.http,
264            probe: self.probe,
265            adapter: self,
266            token: self.token.as_deref(),
267        }
268        .embed(handle, model, inputs, cancel)
269        .await
270    }
271    fn name(&self) -> &'static str {
272        "lmstudio"
273    }
274    fn trusts(&self, identity: &ProcessIdentity) -> bool {
275        trusted_executable(
276            &self.trusted_executables,
277            &identity.executable,
278            cfg!(windows),
279        )
280    }
281    async fn serve(
282        &self,
283        input: &ServeRequest,
284        cancel: &CancellationToken,
285    ) -> Result<ServerState, BackendError> {
286        state(
287            self.name(),
288            input,
289            self.checked(input, cancel).await?,
290            false,
291            None,
292        )
293    }
294    async fn ready(
295        &self,
296        input: &ServeRequest,
297        cancel: &CancellationToken,
298    ) -> Result<(), BackendError> {
299        self.checked(input, cancel).await.map(|_| ())
300    }
301    async fn stop(
302        &self,
303        handle: &ServerState,
304        _cancel: &CancellationToken,
305    ) -> Result<(), BackendError> {
306        handle
307            .validate()
308            .map_err(|error| BackendError(error.to_string()))?;
309        if handle.backend != "lmstudio" || handle.owned_by_us {
310            return Err(BackendError("invalid LM Studio handle".into()));
311        }
312        Ok(())
313    }
314}
315
316pub struct MlxAdapter<'runtime> {
317    pub binary: PathBuf,
318    pub http: &'runtime dyn Transport,
319    pub probe: &'runtime dyn ProcessProbe,
320    pub control: &'runtime dyn ProcessControl,
321    pub commands: &'runtime dyn CommandRunner,
322    pub token: Option<String>,
323}
324/// macOS framework builds run `bin/python3.x` as `Resources/Python.app/Contents/MacOS/Python`.
325pub fn mlx_process_executable(binary: &Path) -> PathBuf {
326    fn name(path: Option<&Path>) -> Option<&str> {
327        path.and_then(Path::file_name)
328            .and_then(|name| name.to_str())
329    }
330    let bin = binary.parent();
331    let version = bin.and_then(Path::parent);
332    let versions = version.and_then(Path::parent);
333    let framework = versions.and_then(Path::parent);
334    let interpreter = name(Some(binary)).is_some_and(|name| {
335        name.strip_prefix("python").is_some_and(|rest| {
336            rest.chars()
337                .all(|character| character.is_ascii_digit() || character == '.')
338        })
339    });
340    match version {
341        Some(version)
342            if interpreter
343                && name(bin) == Some("bin")
344                && name(versions) == Some("Versions")
345                && name(framework) == Some("Python.framework") =>
346        {
347            version.join("Resources/Python.app/Contents/MacOS/Python")
348        }
349        _ => binary.to_path_buf(),
350    }
351}
352pub fn validate_mlx_directory(root: &Path) -> Result<(), BackendError> {
353    if !root.is_absolute()
354        || !root.is_dir()
355        || std::fs::symlink_metadata(root)
356            .map_err(|_| BackendError("model root unavailable".into()))?
357            .file_type()
358            .is_symlink()
359    {
360        return Err(BackendError(
361            "verified absolute MLX directory required".into(),
362        ));
363    }
364    fn scan(path: &Path, count: &mut usize, weights: &mut bool) -> Result<(), BackendError> {
365        for entry in std::fs::read_dir(path)
366            .map_err(|_| BackendError("cannot inspect model directory".into()))?
367        {
368            let entry = entry.map_err(|_| BackendError("cannot inspect model entry".into()))?;
369            *count += 1;
370            if *count > 1024 {
371                return Err(BackendError("model directory exceeds entry limit".into()));
372            }
373            let kind = entry
374                .file_type()
375                .map_err(|_| BackendError("cannot inspect model entry type".into()))?;
376            if kind.is_dir() {
377                scan(&entry.path(), count, weights)?;
378            } else if kind.is_file() {
379                let name = entry.file_name().to_string_lossy().to_ascii_lowercase();
380                if ["py", "pyc", "pyo", "so", "dylib", "dll", "bundle"]
381                    .iter()
382                    .any(|extension| name.ends_with(&format!(".{extension}")))
383                {
384                    return Err(BackendError("executable MLX artifact forbidden".into()));
385                }
386                *weights |= name.ends_with(".safetensors");
387            } else {
388                return Err(BackendError(
389                    "symlink or special MLX artifact forbidden".into(),
390                ));
391            }
392        }
393        Ok(())
394    }
395    let mut count = 0;
396    let mut weights = false;
397    scan(root, &mut count, &mut weights)?;
398    if !weights {
399        return Err(BackendError("MLX weights absent".into()));
400    }
401    for file in ["config.json", "tokenizer_config.json"] {
402        let raw = secure_read(&root.join(file), 1048576, false)
403            .map_err(|_| BackendError("MLX config unavailable".into()))?;
404        let value: Value =
405            serde_json::from_str(&raw).map_err(|_| BackendError("invalid MLX config".into()))?;
406        if !value.is_object()
407            || ["auto_map", "model_file"]
408                .iter()
409                .any(|key| value.get(key).is_some_and(|value| !value.is_null()))
410            || value.get("trust_remote_code") == Some(&Value::Bool(true))
411        {
412            return Err(BackendError("MLX remote code config forbidden".into()));
413        }
414    }
415    Ok(())
416}
417impl MlxAdapter<'_> {
418    async fn ready_once(
419        &self,
420        input: &ServeRequest,
421        cancel: &CancellationToken,
422    ) -> Result<(), BackendError> {
423        let token = self
424            .token
425            .as_deref()
426            .filter(|value| value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()))
427            .ok_or_else(|| BackendError("MLX session token required".into()))?;
428        let health = request(
429            self.http,
430            &input.endpoint,
431            "/health",
432            None,
433            Some(token),
434            cancel,
435        )
436        .await?;
437        if health.get("status").and_then(Value::as_str) != Some("ok") {
438            return Err(BackendError("MLX health not ready".into()));
439        }
440        let path = input
441            .model_path
442            .as_ref()
443            .and_then(|path| path.to_str())
444            .ok_or_else(|| BackendError("MLX model path required".into()))?;
445        if !ids(request(
446            self.http,
447            &input.endpoint,
448            "/v1/models",
449            None,
450            Some(token),
451            cancel,
452        )
453        .await?)?
454        .iter()
455        .any(|id| id == path)
456        {
457            return Err(BackendError("MLX loaded path mismatch".into()));
458        }
459        let completion=request(self.http,&input.endpoint,"/v1/chat/completions",Some(json!({"model":"default_model","messages":[{"role":"user","content":"Reply briefly."}],"max_tokens":1,"stream":false})),Some(token),cancel).await?;
460        if !completion
461            .get("choices")
462            .and_then(Value::as_array)
463            .and_then(|choices| choices.first())
464            .and_then(|choice| choice.get("message"))
465            .and_then(|message| message.get("content"))
466            .is_some_and(Value::is_string)
467        {
468            return Err(BackendError("MLX identity completion failed".into()));
469        }
470        Ok(())
471    }
472}
473#[async_trait::async_trait]
474impl BackendAdapter for MlxAdapter<'_> {
475    async fn chat(
476        &self,
477        handle: &ServerState,
478        input: &crate::ollama_inference::ChatInput,
479        cancel: &CancellationToken,
480    ) -> Result<crate::ollama_inference::ChatResult, BackendError> {
481        crate::openai::OpenAiInference {
482            http: self.http,
483            probe: self.probe,
484            adapter: self,
485            token: None,
486        }
487        .chat(handle, input, cancel)
488        .await
489    }
490    fn name(&self) -> &'static str {
491        "mlx"
492    }
493    fn trusts(&self, identity: &ProcessIdentity) -> bool {
494        self.binary.to_str() == Some(identity.executable.as_str())
495            || mlx_process_executable(&self.binary).to_str() == Some(identity.executable.as_str())
496    }
497    async fn ready_handle(
498        &self,
499        input: &ServeRequest,
500        handle: &ServerState,
501        cancel: &CancellationToken,
502    ) -> Result<(), BackendError> {
503        let session = MlxAdapter {
504            binary: self.binary.clone(),
505            http: self.http,
506            probe: self.probe,
507            control: self.control,
508            commands: self.commands,
509            token: handle.auth_token.clone(),
510        };
511        session.ready(input, cancel).await
512    }
513    async fn serve(
514        &self,
515        input: &ServeRequest,
516        cancel: &CancellationToken,
517    ) -> Result<ServerState, BackendError> {
518        if !cfg!(all(target_os = "macos", target_arch = "aarch64")) {
519            return Err(BackendError("MLX requires Apple Silicon".into()));
520        }
521        let url = validate(input)?;
522        if cancel.is_cancelled() {
523            return Err(BackendError("cancelled".into()));
524        }
525        if self.control.occupied(&input.endpoint).await? {
526            return Err(BackendError(
527                "MLX never attaches to occupied endpoints".into(),
528            ));
529        }
530        let path = input
531            .model_path
532            .as_ref()
533            .ok_or_else(|| BackendError("MLX model path required".into()))?;
534        validate_mlx_directory(path)?;
535        let version=self.commands.run(&self.binary,&["-I".into(),"-c".into(),"import importlib.metadata as m; from mlx_lm import server as s; assert hasattr(s, '_run_http_server') and hasattr(s, 'APIHandler'); print(m.version('mlx-lm'))".into()],cancel,Duration::from_secs(5)).await?;
536        if version.trim() != "0.31.3" {
537            return Err(BackendError("MLX requires audited mlx-lm 0.31.3".into()));
538        }
539        let token = format!(
540            "{}{}",
541            uuid::Uuid::new_v4().simple(),
542            uuid::Uuid::new_v4().simple()
543        );
544        let mut env = minimal_env();
545        env.insert("LLMUP_MLX_AUTH_TOKEN".into(), token.clone());
546        let args = vec![
547            "-I".into(),
548            "-c".into(),
549            MLX_WRAPPER.into(),
550            "mlx_lm.server".into(),
551            "--model".into(),
552            path.to_string_lossy().into_owned(),
553            "--host".into(),
554            url.host_str()
555                .ok_or_else(|| BackendError("missing MLX host".into()))?
556                .trim_matches(['[', ']'])
557                .into(),
558            "--port".into(),
559            url.port_or_known_default()
560                .ok_or_else(|| BackendError("missing MLX port".into()))?
561                .to_string(),
562            "--allowed-origins".into(),
563            String::new(),
564            "--log-level".into(),
565            "ERROR".into(),
566        ];
567        if cancel.is_cancelled() {
568            return Err(BackendError("cancelled".into()));
569        }
570        let child = self
571            .control
572            .spawn(&SpawnSpec {
573                binary: self.binary.clone(),
574                args,
575                env,
576            })
577            .await?;
578        let session = MlxAdapter {
579            binary: self.binary.clone(),
580            http: self.http,
581            probe: self.probe,
582            control: self.control,
583            commands: self.commands,
584            token: Some(token.clone()),
585        };
586        let observed = wait_owned_with_timeout(
587            child,
588            &input.endpoint,
589            mlx_process_executable(&self.binary)
590                .to_str()
591                .ok_or_else(|| BackendError("invalid interpreter path".into()))?,
592            self.probe,
593            cancel,
594            Duration::from_secs(300),
595            || async { session.ready(input, cancel).await.map_err(|error| error.0) },
596        )
597        .await?;
598        state(self.name(), input, observed, true, Some(token))
599    }
600    async fn ready(
601        &self,
602        input: &ServeRequest,
603        cancel: &CancellationToken,
604    ) -> Result<(), BackendError> {
605        validate(input)?;
606        let operation = async {
607            for _ in 0..150 {
608                if self.ready_once(input, cancel).await.is_ok() {
609                    return Ok(());
610                }
611                tokio::time::sleep(Duration::from_millis(250)).await;
612            }
613            Err(BackendError("MLX readiness exhausted".into()))
614        };
615        tokio::select! {biased;_=cancel.cancelled()=>Err(BackendError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(300),operation)=>result.map_err(|_|BackendError("MLX readiness timed out".into()))?}
616    }
617    async fn stop(
618        &self,
619        handle: &ServerState,
620        cancel: &CancellationToken,
621    ) -> Result<(), BackendError> {
622        handle
623            .validate()
624            .map_err(|error| BackendError(error.to_string()))?;
625        if handle.backend != "mlx" || !handle.owned_by_us {
626            return Err(BackendError("invalid owned MLX handle".into()));
627        }
628        let expected = ProcessIdentity {
629            pid: handle
630                .pid
631                .ok_or_else(|| BackendError("missing MLX PID".into()))?,
632            process: String::new(),
633            executable: handle
634                .process_executable
635                .clone()
636                .ok_or_else(|| BackendError("missing MLX executable".into()))?,
637            started: handle
638                .process_started_at
639                .clone()
640                .ok_or_else(|| BackendError("missing MLX start identity".into()))?,
641        };
642        stop_owned(
643            &handle.endpoint,
644            &expected,
645            self.probe,
646            self.control,
647            cancel,
648        )
649        .await
650        .map_err(BackendError)
651    }
652}
653const MLX_WRAPPER: &str = r#"
654import hmac, io, json, os, sys
655from mlx_lm import server as _server
656_original_run = _server._run_http_server
657class GuardedHandler(_server.APIHandler):
658  def _reject(self, status):
659    self.send_response(status)
660    self.send_header("Content-Length", "0")
661    self.end_headers()
662  def do_OPTIONS(self):
663    self._reject(403)
664  def _authenticated(self):
665    return hmac.compare_digest(self.headers.get("Authorization", ""), "Bearer " + os.environ["LLMUP_MLX_AUTH_TOKEN"])
666  def do_GET(self):
667    if not self._authenticated():
668      self._reject(401)
669      return
670    super().do_GET()
671  def do_POST(self):
672    if self.headers.get("Origin") not in (None, "http://127.0.0.1", "http://localhost"):
673      self._reject(403)
674      return
675    if not self._authenticated():
676      self._reject(401)
677      return
678    if self.headers.get("Content-Type", "").split(";", 1)[0].strip().lower() != "application/json":
679      self._reject(415)
680      return
681    try:
682      length = int(self.headers.get("Content-Length", ""))
683    except ValueError:
684      self._reject(411)
685      return
686    if length < 0 or length > 4194304:
687      self._reject(413)
688      return
689    raw = self.rfile.read(length)
690    try:
691      body = json.loads(raw.decode("utf-8"))
692    except (UnicodeDecodeError, json.JSONDecodeError):
693      self._reject(400)
694      return
695    if not isinstance(body, dict) or body.get("model", "default_model") != "default_model":
696      self._reject(400)
697      return
698    maximum = body.get("max_completion_tokens", body.get("max_tokens", 512))
699    if not isinstance(maximum, int) or isinstance(maximum, bool) or not 0 <= maximum <= 4096:
700      self._reject(400)
701      return
702    self.rfile = io.BytesIO(raw)
703    super().do_POST()
704def _guarded_run(host, port, response_generator, server_class=_server.ThreadingHTTPServer, handler_class=GuardedHandler):
705  return _original_run(host, port, response_generator, server_class, GuardedHandler)
706_server._run_http_server = _guarded_run
707sys.argv.pop(1)
708_server.main()
709"#;