Struct SampledTokenClassifier 

pub struct SampledTokenClassifier<'model> { /* private fields */ }

Implementations

impl<'model> SampledTokenClassifier<'model>

pub fn new(model: &'model LlamaModel, markers: StreamingMarkers) -> Self

pub fn ingest(&mut self, token: LlamaToken) -> Vec<IngestOutcome>

pub fn ingest_prompt_token(&mut self, token: LlamaToken)

pub fn ingest_prompt_tokens(&mut self, tokens: &[LlamaToken])

pub fn flush(&mut self) -> Vec<IngestOutcome>

pub fn sample( &mut self, sampler: &mut LlamaSampler, context: &LlamaContext<'_>, idx: i32, ) -> Result<(LlamaToken, Vec<IngestOutcome>), SampleError>

Errors

Forwards LlamaSampler::sample errors verbatim. Nothing is recorded on failure.

Returns the raw sampled token (for downstream batch.add / is_eog_token calls) alongside the outcomes that finalised this turn — see Self::ingest for buffering semantics.

pub fn feed_prompt_to_batch( &mut self, batch: &mut LlamaBatch<'_>, token: LlamaToken, position: llama_pos, seq_ids: &[llama_seq_id], logits: bool, ) -> Result<(), BatchAddError>

Errors

Forwards LlamaBatch::add errors verbatim. Nothing is staged on failure.

pub fn feed_prompt_sequence_to_batch( &mut self, batch: &mut LlamaBatch<'_>, tokens: &[LlamaToken], seq_id: llama_seq_id, logits_all: bool, ) -> Result<(), BatchAddError>

Errors

Forwards LlamaBatch::add_sequence errors verbatim. Nothing is staged on failure.

pub const fn commit_prompt_tokens(&mut self) -> u64

pub const fn discard_pending_prompt_tokens(&mut self) -> u64

pub const fn pending_prompt_tokens(&self) -> u64

pub fn eval_multimodal_chunks( &mut self, chunks: &MtmdInputChunks, mtmd_ctx: &MtmdContext, llama_ctx: &LlamaContext<'_>, start_position: llama_pos, seq_id: llama_seq_id, n_batch: i32, logits_last: bool, ) -> Result<llama_pos, EvalMultimodalChunksError>

Errors

Returns EvalMultimodalChunksError::EvalFailed when the underlying eval_chunks call fails (no counters move), EvalMultimodalChunksError::UnknownChunkType when a chunk reports a type unknown to this binding, or EvalMultimodalChunksError::ChunkOutOfBounds when a valid index returns None from chunks.get.

pub const fn record_prompt_tokens(&mut self, count: u64)

pub const fn record_input_image_tokens(&mut self, count: u64)

pub const fn record_input_audio_tokens(&mut self, count: u64)

pub const fn record_cached_prompt_tokens( &mut self, count: u64, ) -> Result<(), TokenUsageError>

Errors

Forwards TokenUsageError::CachedExceedsPrompt when the running cached total would exceed the prompt total.

pub const fn usage(&self) -> &TokenUsage

pub fn into_usage(self) -> TokenUsage

pub const fn current_section(&self) -> SampledTokenSection

pub const fn markers(&self) -> &StreamingMarkers