libmir-cuda 0.3.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

mod admission;
mod backend;
mod checkpoint;
mod config;
mod engine;
mod error;
pub mod kernels;
mod tensor;

pub use admission::{CudaArchitecture, CudaDecoderRuntime, admit_architecture};
pub use backend::{
    AffineGatedDeltaLayerConfig, AffineGatedDeltaLayerWeights, AffineGatedDeltaMoeLayerConfig,
    AffineGatedFullAttentionConfig, AffineGatedFullAttentionMoeLayerConfig,
    AffineGatedFullAttentionWeights, AffineQuantizedBf16Linear, AffineQuantizedBf16Qmm,
    AffineQuantizedConfig, AffineQuantizedEmbedding, AffineQuantizedPairTensors,
    AffineQuantizedTensors, AffineQuantizedWeight, AffineRouterBf16, AffineSharedExpertMoeConfig,
    AffineSharedExpertMoeWeights, AttentionExecution, AttentionPlan, AttentionPlanRequest,
    BatchedDecodeAttentionBf16, BatchedDecodeMoeBlockBf16, BatchedDecodeMoeLayer,
    BatchedPagedAttentionBf16, BatchedPrefillPagedAttentionBf16, Bf16Embedding, Bf16Linear,
    Bf16LinearPack, Bf16LinearPackWeights, Bf16LinearPair, Bf16LinearPairWeights, Bf16Projection,
    Bf16VectorLinear, Bf16VendorLinear, BlockFp8LinearWeight, BucketedNvFp4MoeBf16,
    CapturedDecodeAttentionBf16, CapturedDecodeMoeBlockBf16, CompressedInt8Bf16Linear,
    CompressedInt8Weight, CudaAffineGatedDeltaExecution, CudaAffineGatedDeltaLayer,
    CudaAffineGatedDeltaMoeExecution, CudaAffineGatedDeltaMoeLayer, CudaAffineGatedFullAttention,
    CudaAffineGatedFullAttentionExecution, CudaAffineGatedFullAttentionMoeExecution,
    CudaAffineGatedFullAttentionMoeLayer, CudaAffineGatedFullAttentionState, CudaAffineOutputHead,
    CudaAffineSharedExpertMoe, CudaAffineSharedExpertMoeExecution, CudaAttentionPolicy,
    CudaBackend, CudaClampedRoutedModelSession, CudaClampedRoutedModelTemplate, CudaDecodeBatch,
    CudaDenseVectorPolicy, CudaDenseVendorPolicy, CudaDenseWeightPolicy, CudaExecutionPlanner,
    CudaGatedDeltaState, CudaHardwareProfile, CudaKernelAdmission, CudaMemoryArchitecture,
    CudaModelSessionConfig, CudaMoeBatchPolicy, CudaMoeFusionPolicy, CudaMoeModelSession,
    CudaMoeModelTemplate, CudaNumericalPolicy, CudaOutputHead, CudaOutputHeadPolicy,
    CudaPlanningPolicy, CudaSharedRoutedLayerState, CudaSharedRoutedModelSession,
    CudaSharedRoutedModelTemplate, CudaTuningConfig, CudaTuningMode, DecodeAttentionBf16,
    DecodeAttentionConfig, DecodeAttentionOutputWeight, DecodeAttentionWeights, DecodeDenseSwiGlu,
    DecodeGraphAction, DecodeMoeBlockBf16, DecodeMoeBlockConfig, DecodeMoeBlockExecutor,
    DecodeMoeBlockWeights, DecodeMoeLayerTemplate, DecodeQkvWeights, DenseDownSource,
    DenseDownWeight, DenseExecution, DenseGateUpSource, DenseGateUpWeights, DenseOutputSource,
    DensePlan, DensePlanRequest, DenseQkvSource, DenseRole, DenseSwiGluConfig,
    DenseSwiGluLayerTemplate, DenseSwiGluWeights, DenseWeightSource, DeviceBatchSamplerBf16,
    DeviceSamplerBf16, DirectFp8Bf16Linear, DirectFp8CheckpointWeight, DirectFp8EmbeddingLookup,
    DirectNvFp4MoeBf16, ExecutionPhase, Fp8ResidualLinearWeight, GatedActivation, GatedDeltaInputs,
    GatedDeltaStateConfig, GroupedNvFp4MoeBf16, HybridNvFp4MoeBf16, MoeExecution, MoePlan,
    MoePlanRequest, MoeQuantization, MxFp4Bf16Linear, MxFp4CheckpointWeight, MxFp4EmbeddingLookup,
    MxFp4ExpertWeights, MxFp4GatheredBf16Linear, MxFp4GatheredMoeBf16, MxFp8Bf16Linear,
    MxFp8CheckpointWeight, MxFp8EmbeddingLookup, MxFp8ExpertWeights, MxFp8GatheredBf16Linear,
    MxFp8GatheredMoeBf16, NvFp4Bf16Linear, NvFp4Bf16Pack, NvFp4Config, NvFp4ExpertBank,
    NvFp4ExpertBankConfig, NvFp4ExpertSource, NvFp4LinearWeight, NvFp4Tensors,
    NvFp4WeightOnlyBf16Linear, NvFp4WeightOnlyWeight, OutputHeadExecution, OutputHeadPlan,
    OutputHeadPlanRequest, PackedIntegerBf16Linear, PackedIntegerWeight, PagedAttentionBf16,
    PagedDecodeBatch, PagedKvCache, PagedPrefillBatch, PlanSource, PrefillAttentionBf16,
    PrefillDenseSwiGlu, PrefillMoeBlockBf16, ProjectionFormat, RmsNormBf16, RopeBf16, RouterBf16,
    RouterSelection, RouterTensors, SelectedAffineGatedBf16Linear, SelectedAffinePairBf16Linear,
    SelectedAffineReduceBf16Linear, SelectedNvFp4LinearBf16, SelectedNvFp4MoeBf16,
    SelectedNvFp4TensorCoreMoeBf16,
};
pub use checkpoint::{
    DenseSwiGluLayerLoadConfig, NvFp4MoeLayerLoadConfig, SharedRoutedModelLoadConfig,
};
pub use config::CudaConfig;
pub use engine::{CudaEngine, CudaGenerationStepOutput, CudaMemoryStats, CudaPrefillBatch};
pub use error::{Error, Result};
pub use kernels::{RopeSpec, RouterSpec};
pub use tensor::{CudaTensor, CudaTensorDType, CudaTensorSet, TensorUploadBatch};