Struct LlamaContextParams 

pub struct LlamaContextParams { /* private fields */ }

Builder for llama_context_params.

Construct with Default::default(), chain with_* setters, then pass the value to crate::model::LlamaModel::new_context. Getter methods mirror the fields that exist on the underlying C struct.

Sampler ownership

Self::with_sampler_seq_configs stores owned LlamaSampler chains inside this struct until the context is created. Clone clears sampler configs because the underlying chains cannot be duplicated safely.

Examples

use llama_cpp_4::context::params::LlamaContextParams;

let ctx_params = LlamaContextParams::default()
    .with_n_ctx(NonZeroU32::new(2048));

assert_eq!(ctx_params.n_ctx(), NonZeroU32::new(2048));

Implementations

impl LlamaContextParams

pub fn with_flash_attn_type(self, flash_attn_type: LlamaFlashAttnType) -> Self

Set the flash-attention mode (Auto, Enabled, or Disabled).

Maps to llama_context_params.flash_attn_type. Use LlamaFlashAttnType::Auto to match llama.cpp defaults.

Examples

use llama_cpp_4::context::params::{LlamaContextParams, LlamaFlashAttnType};
let params = LlamaContextParams::default()
    .with_flash_attn_type(LlamaFlashAttnType::Auto);
assert_eq!(params.flash_attn_type(), LlamaFlashAttnType::Auto);

pub fn flash_attn_type(&self) -> LlamaFlashAttnType

Get the configured flash-attention mode.

pub fn with_attention_type(self, attention_type: LlamaAttentionType) -> Self

Set the attention type used when extracting embeddings.

Maps to llama_context_params.attention_type. Embedding models often need LlamaAttentionType::NonCausal; generative decoding uses LlamaAttentionType::Causal.

Examples

use llama_cpp_4::context::params::{LlamaAttentionType, LlamaContextParams};
let params = LlamaContextParams::default()
    .with_attention_type(LlamaAttentionType::Causal);
assert_eq!(params.attention_type(), LlamaAttentionType::Causal);

pub fn attention_type(&self) -> LlamaAttentionType

Get the attention type used when extracting embeddings.

pub fn with_n_outputs_max(self, n_outputs_max: u32) -> Self

Set the maximum number of outputs per micro-batch.

Maps to llama_context_params.n_outputs_max. When 0, llama.cpp uses n_batch as the cap.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_n_outputs_max(256);
assert_eq!(params.n_outputs_max(), 256);

pub fn n_outputs_max(&self) -> u32

Get the maximum number of outputs per micro-batch.

pub fn with_n_outputs_max_per_seq(self, n_outputs_max_per_seq: u32) -> Self

Set the maximum number of outputs per sequence.

Maps to llama_context_params.n_outputs_max_per_seq. Defaults to 1 — a single output per sequence; pass 0 to fall back to Self::n_outputs_max instead. Backend samplers are initialized for this many outputs per sequence, so multi-output backend sampling (e.g. speculative decoding) must raise it above 1. llama.cpp clamps the value to n_outputs_max.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_n_outputs_max_per_seq(16);
assert_eq!(params.n_outputs_max_per_seq(), 16);

pub fn n_outputs_max_per_seq(&self) -> u32

Get the maximum number of outputs per sequence.

pub fn with_kv_unified(self, kv_unified: bool) -> Self

Use a unified KV buffer across input sequences.

Maps to llama_context_params.kv_unified. Disabling can improve throughput for batched decoding when sequences do not share a long prefix.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_kv_unified(false);
assert!(!params.kv_unified());

pub fn kv_unified(&self) -> bool

Returns true when a unified KV buffer is enabled.

pub fn with_swa_full(self, swa_full: bool) -> Self

Use a full-size sliding-window-attention (SWA) KV cache.

Maps to llama_context_params.swa_full. When false and n_seq_max > 1, llama.cpp may use a smaller per-sequence SWA window for better performance.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_swa_full(true);
assert!(params.swa_full());

pub fn swa_full(&self) -> bool

Returns true when full SWA cache is enabled.

pub fn with_op_offload(self, op_offload: bool) -> Self

Offload eligible host tensor operations to the active device.

Maps to llama_context_params.op_offload.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_op_offload(true);
assert!(params.op_offload());

pub fn op_offload(&self) -> bool

Returns true when host tensor ops are offloaded to device.

pub fn with_ctx_other(self, other: &LlamaContext<'_>) -> Self

Pair this context with another for shared memory or cross-context results.

Maps to llama_context_params.ctx_other. The paired context is returned by crate::context::LlamaContext::ctx_other after creation.

other must remain alive until crate::model::LlamaModel::new_context returns.

Examples

let target = model.new_context(&backend, LlamaContextParams::default())?;
let draft = model.new_context(
    &backend,
    LlamaContextParams::default().with_ctx_other(&target),
)?;

pub fn with_yarn_ext_factor(self, yarn_ext_factor: f32) -> Self

Set YaRN extrapolation mix factor.

Maps to llama_context_params.yarn_ext_factor. Negative values use the model default. Only meaningful when super::RopeScalingType::Yarn is active.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_yarn_ext_factor(1.0);
assert_eq!(params.yarn_ext_factor(), 1.0);

pub fn yarn_ext_factor(&self) -> f32

Get YaRN extrapolation mix factor (yarn_ext_factor).

pub fn with_yarn_attn_factor(self, yarn_attn_factor: f32) -> Self

Set YaRN magnitude scaling factor.

Maps to llama_context_params.yarn_attn_factor.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_yarn_attn_factor(1.0);
assert_eq!(params.yarn_attn_factor(), 1.0);

pub fn yarn_attn_factor(&self) -> f32

Get YaRN magnitude scaling factor (yarn_attn_factor).

pub fn with_yarn_beta_fast(self, yarn_beta_fast: f32) -> Self

Set YaRN low correction dimension (yarn_beta_fast).

Maps to llama_context_params.yarn_beta_fast.

pub fn yarn_beta_fast(&self) -> f32

Get YaRN low correction dimension.

pub fn with_yarn_beta_slow(self, yarn_beta_slow: f32) -> Self

Set YaRN high correction dimension (yarn_beta_slow).

Maps to llama_context_params.yarn_beta_slow.

pub fn yarn_beta_slow(&self) -> f32

Get YaRN high correction dimension.

pub fn with_yarn_orig_ctx(self, yarn_orig_ctx: u32) -> Self

Set YaRN original context size.

Maps to llama_context_params.yarn_orig_ctx. 0 uses the model default.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_yarn_orig_ctx(8192);
assert_eq!(params.yarn_orig_ctx(), 8192);

pub fn yarn_orig_ctx(&self) -> u32

Get YaRN original context size (yarn_orig_ctx).

pub fn with_no_perf(self, no_perf: bool) -> Self

Disable performance timing collection for this context.

Maps to llama_context_params.no_perf. When true, calls such as crate::context::LlamaContext::timings return empty counters.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_no_perf(true);
assert!(params.no_perf());

pub fn no_perf(&self) -> bool

Returns true when perf timings are disabled for this context.

pub fn with_abort_callback( self, callback: ggml_abort_callback, user_data: *mut c_void, ) -> Self

Register an abort callback checked during decode() on CPU backends.

Maps to llama_context_params.abort_callback / abort_callback_data. The callback is invoked periodically during long decodes; return a non-zero value to stop the current operation.

user_data is passed through unchanged and must remain valid for the lifetime of any context created from these params.

pub fn with_sampler_seq_configs( self, configs: impl IntoIterator<Item = (i32, LlamaSampler)>, ) -> Self

Assign per-sequence backend sampler chains.

Maps to llama_context_params.samplers / n_samplers. Each LlamaSampler must be a sampler chain created with llama_sampler_chain_init. The samplers are kept alive inside these params until crate::model::LlamaModel::new_context returns.

Pair sequence ids with the chains that should run when decoding those sequences on the backend.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
use llama_cpp_4::sampling::LlamaSampler;

let chain = LlamaSampler::chain_default(&model)?;
let params = LlamaContextParams::default()
    .with_sampler_seq_configs([(0, chain)]);
assert_eq!(params.n_sampler_seq_configs(), 1);

pub fn n_sampler_seq_configs(&self) -> usize

Number of per-sequence sampler configs attached to these params.

Returns 0 when no chains were set or after Clone (sampler chains are not duplicated).

impl LlamaContextParams

pub fn with_n_ctx(self, n_ctx: Option<NonZeroU32>) -> Self

Set the side of the context

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default();
let params = params.with_n_ctx(NonZeroU32::new(2048));
assert_eq!(params.n_ctx(), NonZeroU32::new(2048));

pub fn n_ctx(&self) -> Option<NonZeroU32>

Get the size of the context.

None if the context size is specified by the model and not the context.

Examples

let params = llama_cpp_4::context::params::LlamaContextParams::default();
assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(512));

pub fn with_n_seq_max(self, n_seq_max: u32) -> Self

Set the maximum number of independent sequence states in the context.

This maps to llama.cpp’s llama_context_params.n_seq_max and must match the highest sequence id used by batched decoding.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default()
    .with_n_seq_max(16);
assert_eq!(params.n_seq_max(), 16);

pub fn n_seq_max(&self) -> u32

Get the configured maximum number of independent sequence states.

pub fn with_n_batch(self, n_batch: u32) -> Self

Set the n_batch

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default()
    .with_n_batch(2048);
assert_eq!(params.n_batch(), 2048);

pub fn n_batch(&self) -> u32

Get the n_batch

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default();
assert_eq!(params.n_batch(), 2048);

pub fn with_n_ubatch(self, n_ubatch: u32) -> Self

Set the n_ubatch

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default()
    .with_n_ubatch(512);
assert_eq!(params.n_ubatch(), 512);

pub fn n_ubatch(&self) -> u32

Get the n_ubatch

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default();
assert_eq!(params.n_ubatch(), 512);

pub fn with_ctx_type(self, ctx_type: LlamaContextType) -> Self

Set the context type (e.g. LlamaContextType::Mtp for the draft context in crate::mtp::MtpSession).

pub fn ctx_type(&self) -> LlamaContextType

Get the configured context type.

pub fn with_n_rs_seq(self, n_rs_seq: u32) -> Self

Set the number of recurrent-state snapshots per sequence (MTP rollback).

Must be >= MtpSessionConfig::n_draft_max on the draft context. See crate::mtp.

pub fn n_rs_seq(&self) -> u32

Get the number of recurrent-state snapshots per sequence used for MTP rollback.

pub fn with_offload_kqv(self, enabled: bool) -> Self

Set the offload_kqv parameter to control offloading KV cache & KQV ops to GPU

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default()
    .with_offload_kqv(false);
assert_eq!(params.offload_kqv(), false);

pub fn offload_kqv(&self) -> bool

Get the offload_kqv parameter

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default();
assert_eq!(params.offload_kqv(), true);

pub fn with_rope_scaling_type(self, rope_scaling_type: RopeScalingType) -> Self

Set the type of rope scaling.

Examples

use llama_cpp_4::context::params::{LlamaContextParams, RopeScalingType};
let params = LlamaContextParams::default()
    .with_rope_scaling_type(RopeScalingType::Linear);
assert_eq!(params.rope_scaling_type(), RopeScalingType::Linear);

pub fn rope_scaling_type(&self) -> RopeScalingType

Get the type of rope scaling.

Examples

let params = llama_cpp_4::context::params::LlamaContextParams::default();
assert_eq!(params.rope_scaling_type(), llama_cpp_4::context::params::RopeScalingType::Unspecified);

pub fn with_rope_freq_base(self, rope_freq_base: f32) -> Self

Set the rope frequency base.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default()
   .with_rope_freq_base(0.5);
assert_eq!(params.rope_freq_base(), 0.5);

pub fn rope_freq_base(&self) -> f32

Get the rope frequency base.

Examples

let params = llama_cpp_4::context::params::LlamaContextParams::default();
assert_eq!(params.rope_freq_base(), 0.0);

pub fn with_rope_freq_scale(self, rope_freq_scale: f32) -> Self

Set the rope frequency scale.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default()
  .with_rope_freq_scale(0.5);
assert_eq!(params.rope_freq_scale(), 0.5);

pub fn rope_freq_scale(&self) -> f32

Get the rope frequency scale.

Examples

let params = llama_cpp_4::context::params::LlamaContextParams::default();
assert_eq!(params.rope_freq_scale(), 0.0);

pub fn n_threads(&self) -> i32

Get the number of threads.

Examples

let params = llama_cpp_4::context::params::LlamaContextParams::default();
assert_eq!(params.n_threads(), 4);

pub fn n_threads_batch(&self) -> i32

Get the number of threads allocated for batches.

Examples

let params = llama_cpp_4::context::params::LlamaContextParams::default();
assert_eq!(params.n_threads_batch(), 4);

pub fn with_n_threads(self, n_threads: i32) -> Self

Set the number of threads.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default()
   .with_n_threads(8);
assert_eq!(params.n_threads(), 8);

pub fn with_n_threads_batch(self, n_threads: i32) -> Self

Set the number of threads allocated for batches.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default()
   .with_n_threads_batch(8);
assert_eq!(params.n_threads_batch(), 8);

pub fn embeddings(&self) -> bool

Check whether embeddings are enabled

Examples

let params = llama_cpp_4::context::params::LlamaContextParams::default();
assert!(!params.embeddings());

pub fn with_embeddings(self, embedding: bool) -> Self

Enable the use of embeddings

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default()
   .with_embeddings(true);
assert!(params.embeddings());

pub fn with_cb_eval(self, cb_eval: ggml_backend_sched_eval_callback) -> Self

Set the evaluation callback.

Examples

extern "C" fn cb_eval_fn(
    t: *mut llama_cpp_sys_4::ggml_tensor,
    ask: bool,
    user_data: *mut std::ffi::c_void,
) -> bool {
    false
}

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default().with_cb_eval(Some(cb_eval_fn));

pub fn with_cb_eval_user_data(self, cb_eval_user_data: *mut c_void) -> Self

Set the evaluation callback user data.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
let params = LlamaContextParams::default();
let user_data = std::ptr::null_mut();
let params = params.with_cb_eval_user_data(user_data);

pub unsafe fn with_tensor_capture(self, capture: &mut TensorCapture) -> Self

Attach a TensorCapture to intercept intermediate tensor outputs during crate::LlamaContext::decode.

Sets cb_eval to copy tensors matching the capture filter (layer outputs, named nodes, prefix, or all). After decode(), read results from the capture — see crate::TensorCapture and crate::context::tensor_capture.

The capture must outlive the context. Call TensorCapture::clear before reusing it on another batch. Prefer Self::with_tensor_transactions, which transfers owned pinned callback state into the context.

Safety

The caller must keep capture at a stable address until after the resulting context is dropped. It must not be accessed concurrently with any context operation that can invoke the callback. The reference accepted here does not remain borrowed by the returned params.

Example

use llama_cpp_4::prelude::*;

fn main() {
    let backend = LlamaBackend::init().unwrap();
    let model = LlamaModel::load_from_file(
        &backend,
        "model.gguf",
        &LlamaModelParams::default(),
    )
    .unwrap();

    let mut capture = TensorCapture::for_layers(&[13, 20, 27]);
    let ctx_params = unsafe {
        LlamaContextParams::default().with_tensor_capture(&mut capture)
    };
    let _ctx = model.new_context(&backend, ctx_params).unwrap();
}

pub fn with_tensor_transactions(self, transactions: TensorTransactions) -> Self

Attaches bounded owned tensor transactions to the context.

The transaction state is pinned before its address is installed in the native callback parameters. On successful context creation ownership moves into crate::LlamaContext and remains there until after llama_free.

Panics

Panics if the linked libllama was not built with the decode-lifecycle hooks patch — an ABI mismatch that would otherwise silently disable the hooks or misread the context-params struct.

pub fn with_cache_type_k(self, ty: GgmlType) -> Self

Set the storage type for the K (key) KV cache tensors.

The default is GgmlType::F16. Quantized types like GgmlType::Q5_0 or GgmlType::Q4_0 reduce VRAM usage significantly; combining them with TurboQuant attention rotation (the default) keeps quality high.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
use llama_cpp_4::quantize::GgmlType;
let params = LlamaContextParams::default()
    .with_cache_type_k(GgmlType::Q5_0);

pub fn cache_type_k(&self) -> ggml_type

Get the K-cache storage type.

pub fn with_cache_type_v(self, ty: GgmlType) -> Self

Set the storage type for the V (value) KV cache tensors.

See with_cache_type_k for details.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
use llama_cpp_4::quantize::GgmlType;
let params = LlamaContextParams::default()
    .with_cache_type_v(GgmlType::Q5_0);

pub fn cache_type_v(&self) -> ggml_type

Get the V-cache storage type.

pub fn with_attn_rot_disabled(self, disabled: bool) -> Self

Control the TurboQuant attention-rotation feature (llama.cpp PR #21038).

By default, llama.cpp applies a Hadamard rotation to Q/K/V tensors before writing them into the KV cache. This significantly improves quantized KV-cache quality at near-zero overhead, and is enabled automatically for models whose head dimension is a power of two.

Set disabled = true to opt out (equivalent to LLAMA_ATTN_ROT_DISABLE=1). The env-var is applied just before the context is created and restored afterwards, so this is safe to call from a single thread.

Examples

use llama_cpp_4::context::params::LlamaContextParams;
// Disable rotation for this context only:
let params = LlamaContextParams::default().with_attn_rot_disabled(true);
assert!(params.attn_rot_disabled());

pub fn attn_rot_disabled(&self) -> bool

Returns true if TurboQuant attention rotation is disabled for this context.

let params = llama_cpp_4::context::params::LlamaContextParams::default();
assert!(!params.attn_rot_disabled());

pub fn with_pooling_type(self, pooling_type: LlamaPoolingType) -> Self

Set the type of pooling.

Examples

use llama_cpp_4::context::params::{LlamaContextParams, LlamaPoolingType};
let params = LlamaContextParams::default()
    .with_pooling_type(LlamaPoolingType::Last);
assert_eq!(params.pooling_type(), LlamaPoolingType::Last);

pub fn pooling_type(&self) -> LlamaPoolingType

Get the type of pooling.

Examples

let params = llama_cpp_4::context::params::LlamaContextParams::default();
assert_eq!(params.pooling_type(), llama_cpp_4::context::params::LlamaPoolingType::Unspecified);

pub fn try_clone(&self) -> Result<Self, ParamsCloneError>

Clone these params, failing when sampler chains are attached.

Prefer this over Clone::clone when you need to detect dropped sampler configuration.

Errors

Returns ParamsCloneError::SamplerChains when per-sequence sampler chains are attached and cannot be duplicated, or ParamsCloneError::TensorTransactions when an owned callback program is attached.

Trait Implementations

impl Clone for LlamaContextParams

Duplicate context params for reuse.

Sampler chains attached via LlamaContextParams::with_sampler_seq_configs are not cloned — the copy clears samplers / n_samplers because the underlying C chains cannot be duplicated safely.

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LlamaContextParams

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for LlamaContextParams

Default parameters for LlamaContext. (as defined in llama.cpp by llama_context_default_params)

use llama_cpp_4::context::params::{LlamaContextParams, RopeScalingType};
let params = LlamaContextParams::default();
assert_eq!(params.n_ctx(), NonZeroU32::new(512), "n_ctx should be 512");
assert_eq!(params.rope_scaling_type(), RopeScalingType::Unspecified);

fn default() -> Self

Returns the “default value” for a type.