encoderfile 0.4.0-rc.1

Distribute and run transformer encoders with a single file.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

use crate::{
    common::{SentenceEmbeddingRequest, SentenceEmbeddingResponse, model_type},
    error::ApiError,
    inference,
    runtime::AppState,
    transforms::SentenceEmbeddingTransform,
};

use super::inference::Inference;

impl Inference for AppState<model_type::SentenceEmbedding> {
    type Input = SentenceEmbeddingRequest;
    type Output = SentenceEmbeddingResponse;

    fn inference(&self, request: impl Into<Self::Input>) -> Result<Self::Output, ApiError> {
        let request = request.into();

        let encodings = self.tokenizer.encode_text(request.inputs)?;

        let transform = SentenceEmbeddingTransform::new(self.transform_str())?;

        let results = inference::sentence_embedding::sentence_embedding(
            self.session.lock(),
            &transform,
            encodings,
        )?;

        Ok(SentenceEmbeddingResponse {
            results,
            model_id: self.config.name.clone(),
            metadata: request.metadata,
        })
    }
}