Struct ModelDeploymentCard 

pub struct ModelDeploymentCard {

    pub display_name: String,
    pub model_info: Option<ModelInfoType>,
    pub tokenizer: Option<TokenizerKind>,
    pub prompt_formatter: Option<PromptFormatterArtifact>,
    pub chat_template_file: Option<PromptFormatterArtifact>,
    pub gen_config: Option<GenerationConfig>,
    pub prompt_context: Option<Vec<PromptContextMixin>>,
    pub last_published: Option<DateTime<Utc>>,
    pub revision: u64,
    pub context_length: u32,
    pub kv_cache_block_size: u32,
    pub migration_limit: u32,
    pub user_data: Option<Value>,
    pub runtime_config: ModelRuntimeConfig,
    /* private fields */
}

Fields

display_name: String

Human readable model name, e.g. “Meta Llama 3.1 8B Instruct”

model_info: Option<ModelInfoType>

Model information

tokenizer: Option<TokenizerKind>

Tokenizer configuration

prompt_formatter: Option<PromptFormatterArtifact>

Prompt Formatter configuration

chat_template_file: Option<PromptFormatterArtifact>

chat template may be stored as a separate file instead of in prompt_formatter.

gen_config: Option<GenerationConfig>

Generation config - default sampling params

prompt_context: Option<Vec<PromptContextMixin>>

Prompt Formatter Config

last_published: Option<DateTime<Utc>>

When this card was last advertised by a worker. None if not yet published.

revision: u64

Incrementing count of how many times we published this card

context_length: u32

Max context (in number of tokens) this model can handle

kv_cache_block_size: u32

Size of a KV cache block - vllm only currently Passed to the engine and the KV router.

migration_limit: u32

How many times a request can be migrated to another worker if the HTTP server lost connection to the current worker.

user_data: Option<Value>

User-defined metadata for custom worker behavior

runtime_config: ModelRuntimeConfig

Implementations

impl ModelDeploymentCard

pub fn builder() -> ModelDeploymentCardBuilder

pub fn with_name_only(name: &str) -> ModelDeploymentCard

Create a ModelDeploymentCard where only the name is filled in.

Single-process setups don’t need an MDC to communicate model details, but it simplifies the code to assume we always have one. This is how you get one in those cases. A quasi-null object: https://en.wikipedia.org/wiki/Null_object_pattern

pub fn expiry_check_period() -> Duration

How often we should check if a model deployment card expired because it’s workers are gone

pub fn load_from_json_file<P: AsRef<Path>>(file: P) -> Result<Self>

Load a model deployment card from a JSON file

pub fn load_from_json_str(contents: &str) -> Result<Self, Error>

Load a model deployment card from a JSON string

pub fn save_to_json_file(&self, file: &str) -> Result<(), Error>

Save the model deployment card to a JSON file

pub fn slug(&self) -> &Slug

pub fn to_json(&self) -> Result<String, Error>

Serialize the model deployment card to a JSON string

pub fn mdcsum(&self) -> String

pub fn is_expired(&self) -> bool

Was this card last published a long time ago, suggesting the worker is gone?

pub fn has_tokenizer(&self) -> bool

Is this a full model card with tokenizer? There are cases where we have a placeholder card (see with_name_only).

pub fn tokenizer_hf(&self) -> Result<HfTokenizer>

pub fn is_gguf(&self) -> bool

pub async fn move_to_nats(&mut self, nats_client: Client) -> Result<()>

Move the files this MDC uses into the NATS object store. Updates the URI’s to point to NATS.

pub async fn delete_from_nats(&mut self, nats_client: Client) -> Result<()>

Delete this card from the key-value store and it’s URLs from the object store

pub fn set_name(&mut self, name: &str)

Allow user to override the name we register this model under. Corresponds to vllm’s --served-model-name.

pub fn load_from_disk( config_path: impl AsRef<Path>, custom_template_path: Option<&Path>, ) -> Result<ModelDeploymentCard>

Build an in-memory ModelDeploymentCard from either:

• a folder containing config.json, tokenizer.json and token_config.json
• a GGUF file With an optional custom template

pub async fn load_from_store( model_slug: &Slug, drt: &DistributedRuntime, ) -> Result<Option<Self>>

Load a ModelDeploymentCard from storage the DistributedRuntime is configured to use. Card should be fully local and ready to use when the call returns.

Trait Implementations

impl Clone for ModelDeploymentCard

fn clone(&self) -> ModelDeploymentCard

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ModelDeploymentCard

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for ModelDeploymentCard

fn default() -> ModelDeploymentCard

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for ModelDeploymentCard

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Display for ModelDeploymentCard

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Serialize for ModelDeploymentCard

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Versioned for ModelDeploymentCard

fn revision(&self) -> u64

fn set_revision(&mut self, revision: u64)