Struct ReloadRequest

pub struct ReloadRequest {
    pub model_path: PathBuf,
    pub lora: Vec<Lora>,
    pub state: Vec<State>,
    pub quant: usize,
    pub quant_type: Quant,
    pub precision: Precision,
    pub token_chunk_size: usize,
    pub max_batch: usize,
    pub embed_device: EmbedDevice,
    pub tokenizer_path: PathBuf,
    pub bnf: BnfOption,
    pub adapter: AdapterOption,
}

Fields

model_path: PathBuf

Path to the model.

lora: Vec<Lora>

List of LoRA blended on the model.

state: Vec<State>

Path to the initial state.

quant: usize

Specify layers that needs to be quantized.

quant_type: Quant

Quantization type (Int8 or NF4).

precision: Precision

Precision for intermediate tensors (Fp16 or Fp32).

token_chunk_size: usize

Maximum tokens to be processed in parallel at once.

max_batch: usize

Number of states that are cached on GPU.

embed_device: EmbedDevice

Device to put the embed tensor.

tokenizer_path: PathBuf

Path to the tokenizer.

bnf: BnfOption

BNF options.

adapter: AdapterOption

Adapter selection.

Trait Implementations

impl Clone for ReloadRequest

fn clone(&self) -> ReloadRequest

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ReloadRequest

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for ReloadRequest

fn default() -> Self

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for ReloadRequest

where ReloadRequest: Default,

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for ReloadRequest

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl ToSchema for ReloadRequest

fn to_schema(components: &mut Components) -> RefOr<Schema>

Returns a tuple of name and schema or reference to a schema that can be referenced by the name or inlined directly to responses, request bodies or parameters.