Struct GenerateOptions 

pub struct GenerateOptions {

    pub max_tokens: usize,
    pub temperature: f64,
    pub top_p: Option<f64>,
    pub top_k: Option<usize>,
    pub repeat_penalty: f32,
    pub repeat_last_n: usize,
    pub batch_size: usize,
    pub seed: u64,
    pub system_prompt: Option<String>,
    pub max_batch_size: usize,
    pub batch_window_ms: u64,
    pub enable_prefix_cache: bool,
    pub cache_memory_mb: usize,
    pub cpu_threads: usize,
    pub reserve_cores: usize,
    pub simd_level: String,
}

Configuration options for text generation.

Example

use oxide_rs::GenerateOptions;

let options = GenerateOptions {
    max_tokens: 512,
    temperature: 0.3,
    top_p: None,
    top_k: None,
    repeat_penalty: 1.1,
    repeat_last_n: 64,
    seed: 299792458,
    system_prompt: None,
};

Fields

max_tokens: usize

Maximum number of tokens to generate.

Default: 512

temperature: f64

Sampling temperature. Higher values produce more diverse output, lower values produce more focused output.

Set to 0.0 for greedy/argmax sampling.

Default: 0.3

top_p: Option<f64>

Nucleus sampling (top-p) threshold. Limits sampling to the smallest set of tokens whose cumulative probability exceeds this threshold.

Default: None

top_k: Option<usize>

Top-k sampling. Limits sampling to the k most likely tokens.

Default: None

repeat_penalty: f32

Penalty applied to repeated tokens. Values > 1.0 reduce repetition.

Default: 1.1

repeat_last_n: usize

Number of previous tokens to consider for repeat penalty.

Default: 64

batch_size: usize

Batch size for warmup/prefill.

Default: 128

seed: u64

Random seed for reproducibility. Same seed + same input = same output.

Default: 299792458

system_prompt: Option<String>

System prompt to prepend to the conversation.

Default: None

max_batch_size: usize

Maximum batch size for dynamic batching.

Default: 4

batch_window_ms: u64

Time window (in ms) to wait for batching requests.

Default: 1

enable_prefix_cache: bool

Enable prefix caching for faster TTFT.

Default: true

cache_memory_mb: usize

Memory budget for prefix cache (in MB).

Default: 512

cpu_threads: usize

Number of CPU threads (0 = auto-detect, use n-1).

Default: 0 (auto)

reserve_cores: usize

Number of cores to reserve for OS.

Default: 0

simd_level: String

SIMD level (auto, avx512, avx2, neon, scalar).

Default: auto

Trait Implementations

impl Clone for GenerateOptions

fn clone(&self) -> GenerateOptions

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for GenerateOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for GenerateOptions

fn default() -> Self

Returns the “default value” for a type.