oxillama-runtime 0.1.3

Inference engine — KV cache, sampling, tokenizer bridge
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

//! # oxillama-runtime
//!
//! Inference runtime for OxiLLaMa.
//!
//! Orchestrates the complete inference pipeline: model loading, tokenization,
//! forward pass execution, KV caching, and token sampling.

pub mod batched_attention;
pub mod beam_search;
pub mod embedding;
pub mod engine;
pub mod error;
pub mod flash_attention;
pub mod kv_cache;
pub mod kv_pool;
pub mod lora_loader;
pub mod metrics;
pub mod offload;
pub mod sampling;
pub mod scheduler;
pub mod sequence_pool;
pub mod snapshot;
pub mod speculative;
#[cfg(feature = "native-async")]
pub mod speculative_async;
pub mod tokenizer_bridge;
pub mod tool_dispatch;

pub use batched_attention::batched_flash_attention;
pub use beam_search::{
    beam_generate, BeamForwardPass, BeamHypothesis, BeamSearchConfig, EngineBeamAdapter,
};
pub use embedding::PoolingMode;
pub use engine::{EngineConfig, InferenceEngine, FLASH_ATTN_THRESHOLD};
pub use error::{RuntimeError, RuntimeResult};
pub use flash_attention::{
    flash_attention, flash_attention_forward, flash_attention_gqa, flash_attention_multi_head,
    FlashAttentionConfig,
};
pub use kv_cache::prefix::{CachedKvState, PrefixCacheConfig, PrefixKvCache};
pub use kv_cache::{BatchedKvView, KvCache, KvCacheSnapshot, KvSlot, VecBatchedKvView};
pub use kv_pool::KvCachePool;
pub use lora_loader::apply_lora;
pub use metrics::{EngineMetrics, MetricsSnapshot};
pub use offload::{
    FilePagerSource, LayerPager, MemoryPressureProbe, OffloadPolicy, PagerSource, ResidentTensor,
    TensorEntry, TensorId,
};
pub use oxillama_arch::lora::LoadedLora;
pub use oxillama_arch::LoraStack;
pub use sampling::advanced::{DryStage, EtaStage, TopAStage, TypicalPStage, XtcStage};
pub use sampling::chain::{LogitBias, SamplerChain, SamplerStage};
pub use sampling::grammar::{Grammar, GrammarError, GrammarState, JsonSchemaCompiler};
pub use sampling::{sample, Sampler, SamplerConfig};
pub use scheduler::{Scheduler, SchedulerConfig, MAX_DECODE_WAIT_MS, PREFILL_CHUNK};
pub use sequence_pool::{PoolError, PoolResult, SequencePool, SequenceSlot, SsmStatePool};
pub use speculative::{SpeculativeConfig, SpeculativeDeltaSync, SpeculativeEngine};
#[cfg(feature = "native-async")]
pub use speculative_async::{
    AsyncSpecConfig, RewindError, Rewindable, SpecStats, SpeculativeDecoder,
};
pub use tool_dispatch::{
    no_op_dispatcher, NoOpDispatcher, ToolCall, ToolCallDetector, ToolCallGrammar, ToolDispatcher,
    ToolResult,
};
// TokenizerBridge is always exported — when neither `tokenizer-wasm` nor
// `tokenizer-onig` is enabled the struct still exists but all methods return
// TokenizerNotAvailable.
pub use tokenizer_bridge::TokenizerBridge;