pub struct ServerOptions {
pub memory_limit_gb: Option<f64>,
pub cache_limit_gb: Option<f64>,
pub max_batch_tokens: usize,
pub viterbi_biases: Option<[f32; 6]>,
pub viterbi_calibration_path: Option<PathBuf>,
pub startup_timeout: Duration,
pub io_timeout: Duration,
pub shutdown_timeout: Duration,
pub prewarm_fixed_trace: bool,
pub log_memory_stats: bool,
}Fields§
§memory_limit_gb: Option<f64>§cache_limit_gb: Option<f64>Independent MLX allocator-cache ceiling. The 0.5 GiB default bounds
retained free buffers on qualified 16 GB Macs; None uses MLX’s
own default cache policy.
max_batch_tokens: usizeMaximum padded tokens in any Metal invocation, including single documents. The 32k default is qualified on a 16 GB M4; larger-memory Macs can tune upward.
viterbi_biases: Option<[f32; 6]>Explicit decoder biases in official calibration-file order. When
omitted, the client loads viterbi_calibration.json beside the model.
viterbi_calibration_path: Option<PathBuf>Optional calibration artifact path. This conflicts with explicit
viterbi_biases; when omitted, the model’s sibling artifact is used.
startup_timeout: DurationMaximum time for model import and the startup readiness check.
io_timeout: DurationMaximum time for each inference protocol request.
shutdown_timeout: DurationMaximum graceful shutdown wait before the child is killed.
prewarm_fixed_trace: boolCompile and execute the configured fixed trace before startup returns.
This has no effect on dynamic spawn and can be disabled when cold
startup latency matters more than first-request predictability.
log_memory_stats: boolEmit per-request active, peak, and cached MLX bytes on server stderr.
Trait Implementations§
Source§impl Clone for ServerOptions
impl Clone for ServerOptions
Source§impl Debug for ServerOptions
impl Debug for ServerOptions
Auto Trait Implementations§
impl Freeze for ServerOptions
impl RefUnwindSafe for ServerOptions
impl Send for ServerOptions
impl Sync for ServerOptions
impl Unpin for ServerOptions
impl UnsafeUnpin for ServerOptions
impl UnwindSafe for ServerOptions
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more