ferrum-models 0.11.0

Model architectures (LLaMA, Qwen, BERT) for Ferrum inference
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

//! Metadata-only helpers for existing family fixtures. These deliberately do
//! not claim device compatibility; compiler/runtime tests supply real catalogs.

use ferrum_interfaces::vnext::{
    ModelFamilyRegistration, NumericalExecutionPolicy, PreparedModelFamily, VNextError,
};

use super::{DefinedProductionModel, PreparedProductionModel};

pub(super) trait PrepareFamilyFixture: ModelFamilyRegistration {
    fn prepare_fixture(&self, raw: &serde_json::Value) -> Result<PreparedModelFamily, VNextError> {
        let definition = self.define(raw)?;
        let candidates = definition.numerical_profiles().candidates(
            &NumericalExecutionPolicy::Auto,
            ferrum_types::KvStorageFormat::F16,
        )?;
        self.prepare(&definition, &candidates[0].id)
    }
}
impl<R: ModelFamilyRegistration> PrepareFamilyFixture for R {}

pub(super) fn prepare_product_fixture(
    defined: DefinedProductionModel,
) -> ferrum_types::Result<PreparedProductionModel> {
    let candidates = defined
        .definition()
        .numerical_profiles()
        .candidates(
            &NumericalExecutionPolicy::Auto,
            ferrum_types::KvStorageFormat::F16,
        )
        .map_err(|error| ferrum_types::FerrumError::model(error.to_string()))?;
    defined.prepare(&candidates[0].id)
}