use crate::
{
Client,
environment::HuggingFaceEnvironmentImpl,
secret::Secret,
error::{ Result, HuggingFaceError },
components::
{
input::InferenceParameters,
inference_shared::InferenceResponse,
},
token_counter::{ TokenCounter, CountingStrategy },
cache::{ Cache, CacheConfig },
};
use std::sync::Arc;
use tokio::sync::mpsc;
pub struct SyncStream
{
receiver : mpsc::Receiver< Result< String > >,
runtime : Arc< tokio::runtime::Runtime >,
}
impl Iterator for SyncStream
{
type Item = Result< String >;
#[ inline ]
fn next( &mut self ) -> Option< Self::Item >
{
self.runtime.block_on( async
{
self.receiver.recv().await
} )
}
}
impl core::fmt::Debug for SyncStream
{
#[ inline ]
fn fmt( &self, f : &mut core::fmt::Formatter< '_ > ) -> core::fmt::Result
{
f.debug_struct( "SyncStream" )
.field( "receiver", &"< Receiver >" )
.field( "runtime", &"< Runtime >" )
.finish()
}
}
pub struct SyncCache< K, V >
{
cache : Cache< K, V >,
runtime : Arc< tokio::runtime::Runtime >,
}
impl< K, V > SyncCache< K, V >
where
K : Eq + core::hash::Hash + Clone,
V : Clone,
{
#[ inline ]
pub fn insert( &self, key : K, value : V, ttl : Option< core::time::Duration > )
{
let cache = self.cache.clone();
self.runtime.block_on( async move
{
cache.insert( key, value, ttl ).await;
} );
}
#[ inline ]
pub fn get( &self, key : &K ) -> Option< V >
{
let cache = self.cache.clone();
let key = key.clone();
self.runtime.block_on( async move
{
cache.get( &key ).await
} )
}
#[ inline ]
pub fn contains_key( &self, key : &K ) -> bool
{
let cache = self.cache.clone();
let key = key.clone();
self.runtime.block_on( async move
{
cache.contains_key( &key ).await
} )
}
#[ inline ]
pub fn remove( &self, key : &K ) -> Option< V >
{
let cache = self.cache.clone();
let key = key.clone();
self.runtime.block_on( async move
{
cache.remove( &key ).await
} )
}
#[ inline ]
pub fn clear( &self )
{
let cache = self.cache.clone();
self.runtime.block_on( async move
{
cache.clear().await;
} );
}
#[ inline ]
#[ must_use ]
pub fn len( &self ) -> usize
{
let cache = self.cache.clone();
self.runtime.block_on( async move
{
cache.len().await
} )
}
#[ inline ]
#[ must_use ]
pub fn is_empty( &self ) -> bool
{
let cache = self.cache.clone();
self.runtime.block_on( async move
{
cache.is_empty().await
} )
}
}
impl< K, V > core::fmt::Debug for SyncCache< K, V >
{
#[ inline ]
fn fmt( &self, f : &mut core::fmt::Formatter< '_ > ) -> core::fmt::Result
{
f.debug_struct( "SyncCache" )
.field( "cache", &"< Cache >" )
.field( "runtime", &"< Runtime >" )
.finish()
}
}
#[ derive( Debug ) ]
pub struct SyncClient
{
client : Client< HuggingFaceEnvironmentImpl >,
runtime : Arc< tokio::runtime::Runtime >,
}
impl SyncClient
{
#[ inline ]
pub fn new( api_key : String ) -> Result< Self >
{
let secret = Secret::new( api_key );
let env = HuggingFaceEnvironmentImpl::build( secret, None )?;
let client = Client::build( env )?;
let runtime = tokio::runtime::Runtime::new()
.map_err( |e| HuggingFaceError::Generic( format!( "Failed to create runtime : {e}" ) ) )?;
Ok( Self
{
client,
runtime : Arc::new( runtime ),
} )
}
#[ inline ]
pub fn with_base_url( api_key : String, base_url : String ) -> Result< Self >
{
let secret = Secret::new( api_key );
let env = HuggingFaceEnvironmentImpl::build( secret, Some( base_url ) )?;
let client = Client::build( env )?;
let runtime = tokio::runtime::Runtime::new()
.map_err( |e| HuggingFaceError::Generic( format!( "Failed to create runtime : {e}" ) ) )?;
Ok( Self
{
client,
runtime : Arc::new( runtime ),
} )
}
#[ inline ]
#[ must_use ]
pub fn inference( &self ) -> SyncInference
{
SyncInference
{
client : self.client.clone(),
runtime : Arc::clone( &self.runtime ),
}
}
#[ inline ]
#[ must_use ]
pub fn token_counter( &self ) -> TokenCounter
{
TokenCounter::new( CountingStrategy::Estimation )
}
#[ inline ]
#[ must_use ]
pub fn token_counter_with_strategy( &self, strategy : CountingStrategy ) -> TokenCounter
{
TokenCounter::new( strategy )
}
#[ inline ]
#[ must_use ]
pub fn cache< K, V >( &self ) -> SyncCache< K, V >
where
K : Eq + core::hash::Hash + Clone,
V : Clone,
{
SyncCache
{
cache : Cache::new( CacheConfig::default() ),
runtime : Arc::clone( &self.runtime ),
}
}
#[ inline ]
#[ must_use ]
pub fn cache_with_config< K, V >( &self, config : CacheConfig ) -> SyncCache< K, V >
where
K : Eq + core::hash::Hash + Clone,
V : Clone,
{
SyncCache
{
cache : Cache::new( config ),
runtime : Arc::clone( &self.runtime ),
}
}
}
#[ derive( Debug ) ]
pub struct SyncInference
{
client : Client< HuggingFaceEnvironmentImpl >,
runtime : Arc< tokio::runtime::Runtime >,
}
impl SyncInference
{
#[ inline ]
pub fn create< I, M >( &self, inputs : I, model : M ) -> Result< InferenceResponse >
where
I : Into< String >,
M : AsRef< str >,
{
let inference = self.client.inference();
self.runtime.block_on( async move
{
inference.create( inputs, model ).await
})
}
#[ inline ]
pub fn create_with_parameters< I, M >(
&self,
inputs : I,
model : M,
parameters : InferenceParameters
) -> Result< InferenceResponse >
where
I : Into< String >,
M : AsRef< str >,
{
let inference = self.client.inference();
self.runtime.block_on( async move
{
inference.create_with_parameters( inputs, model, parameters ).await
})
}
#[ inline ]
pub fn create_stream< I, M >(
&self,
inputs : I,
model : M,
parameters : InferenceParameters
) -> Result< SyncStream >
where
I : Into< String >,
M : AsRef< str >,
{
let inference = self.client.inference();
let runtime = Arc::clone( &self.runtime );
let receiver = self.runtime.block_on( async move
{
inference.create_stream( inputs, model, parameters ).await
} )?;
Ok( SyncStream
{
receiver,
runtime,
} )
}
}