Struct OpenAIPreprocessor 

pub struct OpenAIPreprocessor { /* private fields */ }

Implementations

impl OpenAIPreprocessor

pub fn new(mdc: ModelDeploymentCard) -> Result<Arc<Self>>

pub fn new_with_parts( mdc: ModelDeploymentCard, formatter: Arc<dyn OAIPromptFormatter>, hf_tokenizer: Tokenizer, ) -> Result<Arc<Self>>

pub fn tokenize(&self, s: &str) -> Result<Encoding>

Encode a string to it’s tokens

pub fn preprocess_request<R: OAIChatLikeRequest + AnnotationsProvider + SamplingOptionsProvider + StopConditionsProvider + OutputOptionsProvider + NvExtProvider>( &self, request: &R, ) -> Result<(PreprocessedRequest, HashMap<String, String>)>

Translate a NvCreateChatCompletionRequest request to a common completion request. Returns both the common completion request and a hashmap of annotations.

Annotations evaluated by this method include:

• formatted_prompt
• token_ids

pub fn builder<R: OAIChatLikeRequest + AnnotationsProvider + SamplingOptionsProvider + StopConditionsProvider + OutputOptionsProvider + NvExtProvider>( &self, request: &R, ) -> Result<PreprocessedRequestBuilder>

pub fn apply_template<R: OAIChatLikeRequest + AnnotationsProvider + SamplingOptionsProvider + StopConditionsProvider + OutputOptionsProvider + NvExtProvider>( &self, request: &R, ) -> Result<Option<String>>

pub fn gather_tokens<R: OAIChatLikeRequest + AnnotationsProvider + SamplingOptionsProvider + StopConditionsProvider + OutputOptionsProvider + NvExtProvider>( &self, request: &R, builder: &mut PreprocessedRequestBuilder, formatted_prompt: Option<String>, ) -> Result<HashMap<String, String>>

pub async fn preprocess_embedding_request( &self, request: &NvCreateEmbeddingRequest, ) -> Result<(PreprocessedEmbeddingRequest, HashMap<String, String>)>

Preprocess an embedding request, handling both text and token ID inputs.

For text inputs, tokenizes the text using the configured tokenizer. For token ID inputs, uses the provided token IDs directly and skips tokenization.

Returns both the preprocessed request and a hashmap of annotations.

pub fn transform_postprocessor_stream<S, Resp>( stream: S, generator: Box<dyn DeltaGeneratorExt<Resp>>, context: Arc<dyn AsyncEngineContext>, ) -> impl Stream<Item = Annotated<Resp>> + Send

where S: Stream<Item = Annotated<BackendOutput>> + Send + 'static, Resp: Send + Sync + 'static + Debug,

pub fn transform_embedding_postprocessor_stream<S>( stream: S, original_request: NvCreateEmbeddingRequest, ) -> impl Stream<Item = Annotated<NvCreateEmbeddingResponse>> + Send

where S: Stream<Item = Annotated<EmbeddingsEngineOutput>> + Send + 'static,

Transform engine embedding output stream to OpenAI embedding response stream

pub fn should_apply_tool_jail( tool_call_parser: Option<&String>, tool_choice: Option<&ChatCompletionToolChoiceOption>, has_tools: bool, ) -> Result<bool, Error>

Determine if we should apply the tool calling jail based on configuration Returns Ok(true) if jail should be applied, Ok(false) if not, or Err if invalid config

pub fn apply_tool_calling_jail<S>( tool_call_parser: String, stream: S, ) -> impl Stream<Item = Annotated<NvCreateChatCompletionStreamResponse>> + Send

where S: Stream<Item = Annotated<NvCreateChatCompletionStreamResponse>> + Send + 'static,

Apply tool calling jail to the stream if needed

Trait Implementations

impl Operator<Context<NvCreateChatCompletionRequest>, Pin<Box<dyn AsyncEngineStream<Annotated<CreateChatCompletionStreamResponse>, Item = Annotated<CreateChatCompletionStreamResponse>>>>, Context<PreprocessedRequest>, Pin<Box<dyn AsyncEngineStream<Annotated<BackendOutput>, Item = Annotated<BackendOutput>>>>> for OpenAIPreprocessor

fn generate<'life0, 'async_trait>( &'life0 self, request: SingleIn<NvCreateChatCompletionRequest>, next: Arc<dyn AsyncEngine<SingleIn<PreprocessedRequest>, ManyOut<Annotated<BackendOutput>>, Error>>, ) -> Pin<Box<dyn Future<Output = Result<ManyOut<Annotated<NvCreateChatCompletionStreamResponse>>, Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

This method is expected to transform the upstream request UpIn to the downstream request DownIn, call the next AsyncEngine with the transformed request, then transform the downstream response DownOut to the upstream response UpOut.

fn into_operator( self: &Arc<Self>, ) -> Arc<PipelineOperator<UpIn, UpOut, DownIn, DownOut>>

where Self: Sized,

impl Operator<Context<NvCreateCompletionRequest>, Pin<Box<dyn AsyncEngineStream<Annotated<NvCreateCompletionResponse>, Item = Annotated<NvCreateCompletionResponse>>>>, Context<PreprocessedRequest>, Pin<Box<dyn AsyncEngineStream<Annotated<BackendOutput>, Item = Annotated<BackendOutput>>>>> for OpenAIPreprocessor

fn generate<'life0, 'async_trait>( &'life0 self, request: SingleIn<NvCreateCompletionRequest>, next: Arc<dyn AsyncEngine<SingleIn<PreprocessedRequest>, ManyOut<Annotated<BackendOutput>>, Error>>, ) -> Pin<Box<dyn Future<Output = Result<ManyOut<Annotated<NvCreateCompletionResponse>>, Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

This method is expected to transform the upstream request UpIn to the downstream request DownIn, call the next AsyncEngine with the transformed request, then transform the downstream response DownOut to the upstream response UpOut.

fn into_operator( self: &Arc<Self>, ) -> Arc<PipelineOperator<UpIn, UpOut, DownIn, DownOut>>

where Self: Sized,

impl Operator<Context<NvCreateEmbeddingRequest>, Pin<Box<dyn AsyncEngineStream<Annotated<NvCreateEmbeddingResponse>, Item = Annotated<NvCreateEmbeddingResponse>>>>, Context<PreprocessedEmbeddingRequest>, Pin<Box<dyn AsyncEngineStream<Annotated<EmbeddingsEngineOutput>, Item = Annotated<EmbeddingsEngineOutput>>>>> for OpenAIPreprocessor

fn generate<'life0, 'async_trait>( &'life0 self, request: SingleIn<NvCreateEmbeddingRequest>, next: Arc<dyn AsyncEngine<SingleIn<PreprocessedEmbeddingRequest>, ManyOut<Annotated<EmbeddingsEngineOutput>>, Error>>, ) -> Pin<Box<dyn Future<Output = Result<ManyOut<Annotated<NvCreateEmbeddingResponse>>, Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

This method is expected to transform the upstream request UpIn to the downstream request DownIn, call the next AsyncEngine with the transformed request, then transform the downstream response DownOut to the upstream response UpOut.

fn into_operator( self: &Arc<Self>, ) -> Arc<PipelineOperator<UpIn, UpOut, DownIn, DownOut>>

where Self: Sized,