Struct InferenceOptions 

pub struct InferenceOptions {
    pub device: Device,
    pub generation_mode: GenerationMode,
    pub max_new_tokens: usize,
    pub temperature: f32,
    pub repetition_penalty: f32,
    pub top_p: f32,
    pub prompt_style: PromptStyle,
    pub max_image_side: Option<u32>,
    pub preload_language_model: bool,
}

Inference settings (device, decode, prompt, optional resize for speed).

Fields

device: Device

generation_mode: GenerationMode

max_new_tokens: usize

temperature: f32

repetition_penalty: f32

top_p: f32

prompt_style: PromptStyle

max_image_side: Option<u32>

Resize so the longest edge is at most this many pixels before patchify (faster).

preload_language_model: bool

Compile LM graphs on LocateAnythingSession::open.

Implementations

impl InferenceOptions

pub fn for_grounding() -> InferenceOptions

Greedy hybrid defaults suited to grounding (processor prompt).

pub fn device(self, device: Device) -> InferenceOptions

pub fn device_name(self, name: &str) -> Result<InferenceOptions, Error>

pub fn max_new_tokens(self, n: usize) -> InferenceOptions

pub fn generation_mode(self, mode: GenerationMode) -> InferenceOptions

pub fn prompt_style(self, style: PromptStyle) -> InferenceOptions

pub fn max_image_side(self, side: u32) -> InferenceOptions

pub fn preload_language_model(self, yes: bool) -> InferenceOptions

Trait Implementations

impl Clone for InferenceOptions

fn clone(&self) -> InferenceOptions

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for InferenceOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.