List of all items

Structs

• Runtime
• benchmark::BenchmarkComparison
• benchmark::BenchmarkConfig
• benchmark::BenchmarkResults
• benchmark::BenchmarkRunner
• benchmark::GenerationResult
• benchmark::InferenceMetrics
• benchmark::UniLLMBackend
• inference::InferenceMetrics
• inference::InferencePipeline
• inference::InferencePipelineBuilder
• inference::Sampler
• kv_cache::KVCache
• kv_cache::LayerKVCache
• model_core::GenerationConfig
• model_core::MemoryRequirements
• model_core::MoEConfig
• model_core::MoEExpert
• model_core::MoELayer
• model_core::ModelRegistry
• model_core::ModelWeights
• model_core::WeightMetadata
• models_v2::arctic::ArcticAttention
• models_v2::arctic::ArcticConfig
• models_v2::arctic::ArcticExpert
• models_v2::arctic::ArcticLayer
• models_v2::arctic::ArcticMLP
• models_v2::arctic::ArcticMoE
• models_v2::arctic::ArcticModelV2
• models_v2::baichuan::BaichuanAttention
• models_v2::baichuan::BaichuanConfig
• models_v2::baichuan::BaichuanLayer
• models_v2::baichuan::BaichuanMLP
• models_v2::baichuan::BaichuanModelV2
• models_v2::bert::BertAttention
• models_v2::bert::BertConfig
• models_v2::bert::BertEmbeddings
• models_v2::bert::BertEncoder
• models_v2::bert::BertIntermediate
• models_v2::bert::BertLayer
• models_v2::bert::BertModelV2
• models_v2::bert::BertOutput
• models_v2::bert::BertPooler
• models_v2::bert::BertSelfAttention
• models_v2::bert::BertSelfOutput
• models_v2::bloom::BLOOMAttention
• models_v2::bloom::BLOOMConfig
• models_v2::bloom::BLOOMLayer
• models_v2::bloom::BLOOMMLP
• models_v2::bloom::BLOOMModelV2
• models_v2::chatglm::ChatGLMAttention
• models_v2::chatglm::ChatGLMConfig
• models_v2::chatglm::ChatGLMLayer
• models_v2::chatglm::ChatGLMMLP
• models_v2::chatglm::ChatGLMModelV2
• models_v2::clip::CLIPAttention
• models_v2::clip::CLIPConfig
• models_v2::clip::CLIPEncoder
• models_v2::clip::CLIPEncoderLayer
• models_v2::clip::CLIPMLP
• models_v2::clip::CLIPModelV2
• models_v2::clip::CLIPTextEmbeddings
• models_v2::clip::CLIPTextTransformer
• models_v2::clip::CLIPVisionEmbeddings
• models_v2::clip::CLIPVisionTransformer
• models_v2::codellama::CodeLlamaAttention
• models_v2::codellama::CodeLlamaConfig
• models_v2::codellama::CodeLlamaLayer
• models_v2::codellama::CodeLlamaMLP
• models_v2::codellama::CodeLlamaModelV2
• models_v2::cogvlm::CogVLMConfig
• models_v2::cogvlm::CogVLMLayer
• models_v2::cogvlm::CogVLMModelV2
• models_v2::dbrx::DbrxAttention
• models_v2::dbrx::DbrxConfig
• models_v2::dbrx::DbrxExpert
• models_v2::dbrx::DbrxLayer
• models_v2::dbrx::DbrxMoE
• models_v2::dbrx::DbrxModelV2
• models_v2::deepseek::DeepSeekAttention
• models_v2::deepseek::DeepSeekConfig
• models_v2::deepseek::DeepSeekLayer
• models_v2::deepseek::DeepSeekMLP
• models_v2::deepseek::DeepSeekModelV2
• models_v2::deepseek_moe::DeepSeekAttention
• models_v2::deepseek_moe::DeepSeekExpert
• models_v2::deepseek_moe::DeepSeekMoEBlock
• models_v2::deepseek_moe::DeepSeekMoEConfig
• models_v2::deepseek_moe::DeepSeekMoELayer
• models_v2::deepseek_moe::DeepSeekMoEModelV2
• models_v2::encodec::EncodecConfig
• models_v2::encodec::EncodecConv1d
• models_v2::encodec::EncodecConvTranspose1d
• models_v2::encodec::EncodecDecoder
• models_v2::encodec::EncodecDownsampleBlock
• models_v2::encodec::EncodecEncoder
• models_v2::encodec::EncodecLSTM
• models_v2::encodec::EncodecModelV2
• models_v2::encodec::EncodecResidualUnit
• models_v2::encodec::EncodecUpsampleBlock
• models_v2::encodec::ResidualVectorQuantizer
• models_v2::falcon::FalconAttention
• models_v2::falcon::FalconConfig
• models_v2::falcon::FalconDecoderLayer
• models_v2::falcon::FalconMLP
• models_v2::falcon::FalconModelV2
• models_v2::florence::FlorenceConfig
• models_v2::florence::FlorenceDecoderLayer
• models_v2::florence::FlorenceLanguageModel
• models_v2::florence::FlorenceModelV2
• models_v2::florence::FlorenceVisionBlock
• models_v2::florence::FlorenceVisionTower
• models_v2::gemma::GemmaAttention
• models_v2::gemma::GemmaConfig
• models_v2::gemma::GemmaLayer
• models_v2::gemma::GemmaMLP
• models_v2::gemma::GemmaModelV2
• models_v2::gpt2::GPT2Attention
• models_v2::gpt2::GPT2Config
• models_v2::gpt2::GPT2Layer
• models_v2::gpt2::GPT2MLP
• models_v2::gpt2::GPT2ModelV2
• models_v2::gptj::GPTJAttention
• models_v2::gptj::GPTJConfig
• models_v2::gptj::GPTJLayer
• models_v2::gptj::GPTJMLP
• models_v2::gptj::GPTJModelV2
• models_v2::gptneox::GPTNeoXAttention
• models_v2::gptneox::GPTNeoXConfig
• models_v2::gptneox::GPTNeoXLayer
• models_v2::gptneox::GPTNeoXMLP
• models_v2::gptneox::GPTNeoXModelV2
• models_v2::granite::GraniteAttention
• models_v2::granite::GraniteConfig
• models_v2::granite::GraniteLayer
• models_v2::granite::GraniteMLP
• models_v2::granite::GraniteModelV2
• models_v2::grok::GrokAttention
• models_v2::grok::GrokConfig
• models_v2::grok::GrokExpert
• models_v2::grok::GrokLayer
• models_v2::grok::GrokMoE
• models_v2::grok::GrokModelV2
• models_v2::hubert::HuBERTConfig
• models_v2::hubert::HuBERTEncoderLayer
• models_v2::hubert::HuBERTModelV2
• models_v2::idefics::IdeficsConfig
• models_v2::idefics::IdeficsLayer
• models_v2::idefics::IdeficsModelV2
• models_v2::internlm::InternLMAttention
• models_v2::internlm::InternLMConfig
• models_v2::internlm::InternLMLayer
• models_v2::internlm::InternLMMLP
• models_v2::internlm::InternLMModelV2
• models_v2::internvl::InternLMAttention
• models_v2::internvl::InternLMLayer
• models_v2::internvl::InternLMMLP
• models_v2::internvl::InternVLConfig
• models_v2::internvl::InternVLModelV2
• models_v2::internvl::InternViTBlock
• models_v2::internvl::InternViTEncoder
• models_v2::jamba::JambaAttentionBlock
• models_v2::jamba::JambaConfig
• models_v2::jamba::JambaExpert
• models_v2::jamba::JambaLayer
• models_v2::jamba::JambaMambaBlock
• models_v2::jamba::JambaMoE
• models_v2::jamba::JambaModelV2
• models_v2::llama::LlamaAttention
• models_v2::llama::LlamaConfig
• models_v2::llama::LlamaLayer
• models_v2::llama::LlamaMLP
• models_v2::llama::LlamaModelV2
• models_v2::llava::LLaVAConfig
• models_v2::llava::LLaVALanguageAttention
• models_v2::llava::LLaVALanguageLayer
• models_v2::llava::LLaVALanguageMLP
• models_v2::llava::LLaVALanguageModel
• models_v2::llava::LLaVAModelV2
• models_v2::llava::LLaVAMultiModalProjector
• models_v2::llava::LLaVAVisionAttention
• models_v2::llava::LLaVAVisionEmbeddings
• models_v2::llava::LLaVAVisionEncoder
• models_v2::llava::LLaVAVisionLayer
• models_v2::llava::LLaVAVisionMLP
• models_v2::llava::LLaVAVisionTower
• models_v2::mamba::MambaBackbone
• models_v2::mamba::MambaBlock
• models_v2::mamba::MambaConfig
• models_v2::mamba::MambaMixer
• models_v2::mamba::MambaModelV2
• models_v2::mamba::MambaState
• models_v2::minicpm::MiniCPMAttention
• models_v2::minicpm::MiniCPMConfig
• models_v2::minicpm::MiniCPMDecoderLayer
• models_v2::minicpm::MiniCPMMLP
• models_v2::minicpm::MiniCPMModelV2
• models_v2::mistral::MistralAttention
• models_v2::mistral::MistralConfig
• models_v2::mistral::MistralLayer
• models_v2::mistral::MistralMLP
• models_v2::mistral::MistralModelV2
• models_v2::mixtral::MixtralAttention
• models_v2::mixtral::MixtralConfig
• models_v2::mixtral::MixtralExpert
• models_v2::mixtral::MixtralLayer
• models_v2::mixtral::MixtralMoE
• models_v2::mixtral::MixtralModelV2
• models_v2::mpt::MPTAttention
• models_v2::mpt::MPTBlock
• models_v2::mpt::MPTConfig
• models_v2::mpt::MPTFFN
• models_v2::mpt::MPTModelV2
• models_v2::musicgen::MusicGenConfig
• models_v2::musicgen::MusicGenDecoderLayer
• models_v2::musicgen::MusicGenModelV2
• models_v2::olmo::OlmoAttention
• models_v2::olmo::OlmoConfig
• models_v2::olmo::OlmoLayer
• models_v2::olmo::OlmoMLP
• models_v2::olmo::OlmoModelV2
• models_v2::opt::OPTAttention
• models_v2::opt::OPTConfig
• models_v2::opt::OPTLayer
• models_v2::opt::OPTModelV2
• models_v2::phi3_vision::Phi3VisionConfig
• models_v2::phi3_vision::Phi3VisionDecoderLayer
• models_v2::phi3_vision::Phi3VisionEncoder
• models_v2::phi3_vision::Phi3VisionModelV2
• models_v2::phi3_vision::Phi3VisionProjector
• models_v2::phi3_vision::VitBlock
• models_v2::phi::PhiAttention
• models_v2::phi::PhiConfig
• models_v2::phi::PhiLayer
• models_v2::phi::PhiMLP
• models_v2::phi::PhiModelV2
• models_v2::qwen2_vl::PatchEmbedding3D
• models_v2::qwen2_vl::Qwen2VLAttention
• models_v2::qwen2_vl::Qwen2VLConfig
• models_v2::qwen2_vl::Qwen2VLDecoderLayer
• models_v2::qwen2_vl::Qwen2VLLanguageModel
• models_v2::qwen2_vl::Qwen2VLMLP
• models_v2::qwen2_vl::Qwen2VLModelV2
• models_v2::qwen2_vl::Qwen2VLProjector
• models_v2::qwen2_vl::Qwen2VisionEncoder
• models_v2::qwen2_vl::VisionAttention
• models_v2::qwen2_vl::VisionMLP
• models_v2::qwen2_vl::VisionMerger
• models_v2::qwen2_vl::VisionTransformerBlock
• models_v2::qwen::QwenAttention
• models_v2::qwen::QwenConfig
• models_v2::qwen::QwenLayer
• models_v2::qwen::QwenMLP
• models_v2::qwen::QwenModelV2
• models_v2::recurrent_gemma::GriffinAttention
• models_v2::recurrent_gemma::GriffinMLP
• models_v2::recurrent_gemma::GriffinRecurrent
• models_v2::recurrent_gemma::RecurrentGemmaConfig
• models_v2::recurrent_gemma::RecurrentGemmaLayer
• models_v2::recurrent_gemma::RecurrentGemmaModelV2
• models_v2::recurrent_gemma::RecurrentGemmaState
• models_v2::rwkv4::Rwkv4Block
• models_v2::rwkv4::Rwkv4ChannelMixing
• models_v2::rwkv4::Rwkv4Config
• models_v2::rwkv4::Rwkv4ModelV2
• models_v2::rwkv4::Rwkv4State
• models_v2::rwkv4::Rwkv4TimeMixing
• models_v2::rwkv6::Rwkv6Block
• models_v2::rwkv6::Rwkv6ChannelMixing
• models_v2::rwkv6::Rwkv6Config
• models_v2::rwkv6::Rwkv6ModelV2
• models_v2::rwkv6::Rwkv6State
• models_v2::rwkv6::Rwkv6TimeMixing
• models_v2::starcoder::StarCoderAttention
• models_v2::starcoder::StarCoderConfig
• models_v2::starcoder::StarCoderLayer
• models_v2::starcoder::StarCoderMLP
• models_v2::starcoder::StarCoderModelV2
• models_v2::t5::T5Attention
• models_v2::t5::T5Block
• models_v2::t5::T5Config
• models_v2::t5::T5LayerCrossAttention
• models_v2::t5::T5LayerFF
• models_v2::t5::T5LayerSelfAttention
• models_v2::t5::T5ModelV2
• models_v2::t5::T5Stack
• models_v2::wav2vec2::Wav2Vec2Config
• models_v2::wav2vec2::Wav2Vec2ConvLayer
• models_v2::wav2vec2::Wav2Vec2Encoder
• models_v2::wav2vec2::Wav2Vec2EncoderLayer
• models_v2::wav2vec2::Wav2Vec2FeatureExtractor
• models_v2::wav2vec2::Wav2Vec2ModelV2
• models_v2::whisper::WhisperAttention
• models_v2::whisper::WhisperConfig
• models_v2::whisper::WhisperDecoder
• models_v2::whisper::WhisperDecoderLayer
• models_v2::whisper::WhisperEncoder
• models_v2::whisper::WhisperEncoderLayer
• models_v2::whisper::WhisperModelV2
• models_v2::yi::YiAttention
• models_v2::yi::YiConfig
• models_v2::yi::YiLayer
• models_v2::yi::YiMLP
• models_v2::yi::YiModelV2
• ollama::LayerInfo
• ollama::Manifest
• ollama::ModelInfo
• ollama::OllamaRegistry
• precompute::CausalMaskCache
• precompute::RoPECache
• precompute::SlidingWindowMaskCache
• sampler::GreedySampler
• simple_observability::HealthMonitor
• simple_observability::HealthStatus
• simple_observability::MetricsSnapshot
• simple_observability::RequestTimer
• simple_observability::SimpleMetrics
• tensor_core::CandleStorage
• tensor_core::CpuStorage
• tensor_core::CpuTensorOpsImpl
• tensor_core::GpuStorage
• tensor_core::GpuTensorOpsImpl
• tensor_core::Tensor
• tensor_core::TensorDispatcher
• tokenizer::BatchTokenizer
• tokenizer::SpecialTokens
• tokenizer::Tokenizer
• types::GenerationStats
• types::InferenceInputs
• types::InferenceOutput
• types::MemoryRequirements
• types::ModelConfig
• types::ModelOutput
• types::ModelWeightMetadata
• types::ModelWeights
• types::PreparedInputs
• types::Tensor
• types::WeightMetadata
• weight_loader_core::ConfigLoader
• weight_loader_core::GGUFModelConfig
• weight_loader_core::GGUFSpecialTokens
• weight_loader_core::GGUFTokenizer
• weight_loader_core::GGUFWeightLoader
• weight_loader_core::ModelLoader
• weight_loader_core::PyTorchWeightLoader
• weight_loader_core::SafeTensorsWeightLoader
• weight_loader_core::UnifiedWeightLoader

Enums

• model_core::ModelInputs
• model_core::ModelOutputs
• model_core::RouterType
• model_core::WeightFormat
• models_v2::jamba::JambaLayerType
• models_v2::recurrent_gemma::RecurrentGemmaLayerType
• tensor_core::DataType
• tensor_core::Device
• types::DataType
• types::Device
• types::ModelError
• types::ModelFeature
• types::ModelFormat
• types::ModelInputs
• types::ModelOutputs
• types::ModelPrecision
• types::NormalizationType
• types::PositionEmbeddingType
• types::TensorData
• types::WeightFormat

Traits

• benchmark::InferenceBackend
• model_core::MLPLayer
• model_core::Model
• model_core::ModelConfig
• model_core::ModelFactory
• tensor_core::TensorOps
• tensor_core::TensorStorage
• weight_loader_core::WeightLoader

Macros

• model_config

Functions

• benchmark::get_process_memory
• benchmark::print_comparison_table
• benchmark::print_single_results
• model_core::registry
• simple_observability::start_inference_span
• simple_observability::start_request_span
• tensor_core::ops
• tensor_core::ops_fn::add
• tensor_core::ops_fn::attention
• tensor_core::ops_fn::causal_mask
• tensor_core::ops_fn::causal_sliding_window_mask
• tensor_core::ops_fn::clamp
• tensor_core::ops_fn::concat
• tensor_core::ops_fn::conv1d
• tensor_core::ops_fn::embedding
• tensor_core::ops_fn::exp
• tensor_core::ops_fn::flash_attention
• tensor_core::ops_fn::fused_residual_rms_norm
• tensor_core::ops_fn::fused_swiglu
• tensor_core::ops_fn::gather
• tensor_core::ops_fn::gelu
• tensor_core::ops_fn::layer_norm
• tensor_core::ops_fn::matmul
• tensor_core::ops_fn::mul
• tensor_core::ops_fn::normalize
• tensor_core::ops_fn::randn
• tensor_core::ops_fn::rms_norm
• tensor_core::ops_fn::scale
• tensor_core::ops_fn::scatter
• tensor_core::ops_fn::sigmoid
• tensor_core::ops_fn::silu
• tensor_core::ops_fn::sliding_window_mask
• tensor_core::ops_fn::softmax
• tensor_core::ops_fn::sub
• tensor_core::ops_fn::tanh
• tensor_core::ops_fn::topk
• tensor_core::ops_fn::transpose
• tensor_core::ops_fn::transpose_dims
• tensor_core::ops_fn::zeros
• types::calculate_strides
• weight_loader_core::loader

Type Aliases

• types::ModelResult

Statics

• simple_observability::METRICS

Constants

• kv_cache::DEFAULT_MAX_SEQ_LEN
• tokenizer::BOS_TOKEN_ID
• tokenizer::EOS_TOKEN_ID
• tokenizer::PAD_TOKEN_ID
• tokenizer::UNK_TOKEN_ID