encoderfile 0.4.0-rc.1

Distribute and run transformer encoders with a single file.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

use crate::{
    common::{EmbeddingRequest, EmbeddingResponse, model_type},
    error::ApiError,
    inference,
    runtime::AppState,
    transforms::EmbeddingTransform,
};

use super::inference::Inference;

impl Inference for AppState<model_type::Embedding> {
    type Input = EmbeddingRequest;
    type Output = EmbeddingResponse;

    fn inference(&self, request: impl Into<Self::Input>) -> Result<Self::Output, ApiError> {
        let request = request.into();

        let encodings = self.tokenizer.encode_text(request.inputs)?;

        let transform = EmbeddingTransform::new(self.transform_str())?;

        let results = inference::embedding::embedding(self.session.lock(), &transform, encodings)?;

        Ok(EmbeddingResponse {
            results,
            model_id: self.config.name.clone(),
            metadata: request.metadata,
        })
    }
}