use base64::Engine as _;
use futures::stream::BoxStream;
use rs_ai_core::{
AiError, AiResult, CacheConfig, ContentPart, FileData, GenerateOptions, ImageData,
ImageGenerationOptions, ImageModel, ImageResult, LanguageModel, Message, Prompt,
RealtimeSession, StreamEvent, VideoGenerationOptions, VideoModel, VideoResult,
};
use rs_ai_providers::chatgpt::ChatGptProvider;
use rs_ai_providers::claude::ClaudeProvider;
use rs_ai_providers::cloudflare::CloudflareProvider;
use rs_ai_providers::gemini::GeminiProvider;
use rs_ai_providers::openai_compatible::{OpenAiCompatibleConfig, OpenAiCompatibleProvider};
use rs_ai_providers::xai::XaiProvider;
pub use rs_ai_providers::xai::XAI_OAUTH_MODEL_IDS;
pub struct ClientBuilder {
provider_type: ProviderType,
api_key: Option<String>,
model_id: Option<String>,
images: Vec<String>,
cf_gateway: Option<String>,
cache_config: Option<CacheConfig>,
}
#[derive(Debug)]
enum ProviderType {
Claude,
ChatGpt,
Gemini,
Xai,
Cloudflare { account_id: String },
Compatible { base_url: String },
}
impl ClientBuilder {
pub fn api_key(mut self, key: impl Into<String>) -> Self {
self.api_key = Some(key.into());
self
}
pub fn model(mut self, id: impl Into<String>) -> Self {
self.model_id = Some(id.into());
self
}
pub fn with_image(mut self, url_or_path: impl Into<String>) -> Self {
self.images.push(url_or_path.into());
self
}
pub fn cf_ai_gateway(mut self, gateway: impl Into<String>) -> Self {
self.cf_gateway = Some(gateway.into());
self
}
pub fn with_cache(mut self, config: CacheConfig) -> Self {
self.cache_config = Some(config);
self
}
pub fn enable_cache(self) -> Self {
self.with_cache(CacheConfig::new())
}
pub fn with_xai_conv_id(mut self, conv_id: impl Into<String>) -> Self {
let mut cache = self.cache_config.unwrap_or_default();
cache.xai_conv_id = Some(conv_id.into());
self.cache_config = Some(cache);
self
}
pub fn with_prompt_cache_key(mut self, key: impl Into<String>) -> Self {
let mut cache = self.cache_config.unwrap_or_default();
cache.prompt_cache_key = Some(key.into());
self.cache_config = Some(cache);
self
}
pub async fn generate(self, prompt: impl Into<String>) -> AiResult<String> {
let text = prompt.into();
let images = self.images.clone();
let client = self.build().await?;
if images.is_empty() {
client.generate(text).await
} else {
client.generate_with_images(text, images).await
}
}
pub async fn generate_prompt(self, prompt: Prompt) -> AiResult<String> {
if !self.images.is_empty() {
return Err(AiError::ProviderError {
provider: "rs_ai".to_string(),
status: None,
message: "Images cannot be combined with a structured prompt".to_string(),
});
}
self.build().await?.generate_prompt(prompt).await
}
pub async fn stream(
self,
prompt: impl Into<String>,
) -> AiResult<BoxStream<'static, AiResult<StreamEvent>>> {
let text = prompt.into();
let images = self.images.clone();
let client = self.build().await?;
if images.is_empty() {
client.stream(text).await
} else {
client.stream_with_images(text, images).await
}
}
pub async fn stream_prompt(
self,
prompt: Prompt,
) -> AiResult<BoxStream<'static, AiResult<StreamEvent>>> {
if !self.images.is_empty() {
return Err(AiError::ProviderError {
provider: "rs_ai".to_string(),
status: None,
message: "Images cannot be combined with a structured prompt".to_string(),
});
}
self.build().await?.stream_prompt(prompt).await
}
pub async fn speak(self, text: impl Into<String>) -> AiResult<Vec<u8>> {
let text_str = text.into();
let options = GenerateOptions::default();
let model_id_hint = self.model_id.clone().unwrap_or_default();
let client = self.build().await?;
let result = client
.model
.as_ref()
.as_ref()
.generate(Prompt::Text(text_str.clone()), options)
.await?;
if let Some(audio_b64) = result.metadata.extra.get("audio_bytes") {
if let Some(encoded) = audio_b64.as_str() {
let bytes = base64::engine::general_purpose::STANDARD
.decode(encoded)
.map_err(|e| {
AiError::Serialization(format!(
"Failed to decode audio_bytes from model `{}` metadata: {e}",
model_id_hint
))
})?;
return Ok(bytes);
}
}
let spoken_text = result.text.unwrap_or(text_str);
Ok(spoken_text.into_bytes())
}
pub async fn generate_image(
self,
prompt: impl Into<String>,
options: ImageGenerationOptions,
) -> AiResult<ImageResult> {
let text = prompt.into();
let api_key = self.api_key.clone().ok_or_else(|| AiError::AuthError {
message: "API key not set. Use .api_key() to specify credentials.".to_string(),
})?;
match self.provider_type {
ProviderType::ChatGpt => {
let provider = ChatGptProvider::new(api_key);
let model = provider.image_model(&self.model_id.unwrap_or_default());
model.generate_image(&text, options).await
}
ProviderType::Xai => {
let provider = XaiProvider::new(api_key);
let model = provider.image_model(&self.model_id.unwrap_or_default());
model.generate_image(&text, options).await
}
ProviderType::Gemini => {
let provider = GeminiProvider::new(api_key);
let model = provider.image_model(&self.model_id.unwrap_or_default());
model.generate_image(&text, options).await
}
_ => Err(AiError::UnsupportedCapability {
capability: "image_generation".to_string(),
provider: format!("{:?}", self.provider_type),
}),
}
}
pub async fn generate_video(
self,
prompt: impl Into<String>,
options: VideoGenerationOptions,
) -> AiResult<VideoResult> {
let text = prompt.into();
let api_key = self.api_key.clone().ok_or_else(|| AiError::AuthError {
message: "API key not set. Use .api_key() to specify credentials.".to_string(),
})?;
match self.provider_type {
ProviderType::ChatGpt => {
let provider = ChatGptProvider::new(api_key);
let model = provider.video_model(&self.model_id.unwrap_or_default());
model.generate_video(&text, options).await
}
ProviderType::Gemini => {
let provider = GeminiProvider::new(api_key);
let model = provider.video_model();
model.generate_video(&text, options).await
}
_ => Err(AiError::UnsupportedCapability {
capability: "video_generation".to_string(),
provider: format!("{:?}", self.provider_type),
}),
}
}
pub async fn realtime_session(self) -> AiResult<Box<dyn RealtimeSession>> {
let api_key = self.api_key.clone().ok_or_else(|| AiError::AuthError {
message: "API key not set. Use .api_key() to specify credentials.".to_string(),
})?;
let model_id = self.model_id.clone().unwrap_or_default();
match self.provider_type {
ProviderType::ChatGpt => {
let provider = ChatGptProvider::new(api_key);
provider.unified_realtime_session(&model_id).await
}
ProviderType::Gemini => {
let provider = GeminiProvider::new(api_key);
provider.unified_realtime_session(&model_id).await
}
_ => Err(AiError::UnsupportedCapability {
capability: "realtime_session".to_string(),
provider: format!("{:?}", self.provider_type),
}),
}
}
pub async fn transcribe(self, audio: Vec<u8>, mime_type: &str) -> AiResult<String> {
let encoded = base64::engine::general_purpose::STANDARD.encode(&audio);
let audio_part = ContentPart::File {
data: FileData::Base64 {
media_type: mime_type.to_string(),
data: encoded,
},
};
let instruction_part = ContentPart::Text {
text: "Transcribe the audio in the attached file. Return only the transcription text, no commentary.".to_string(),
};
let message = Message {
role: rs_ai_core::Role::User,
content: vec![instruction_part, audio_part],
name: None,
metadata: std::collections::HashMap::new(),
};
let prompt = Prompt::Messages(vec![message]);
let model_id_hint = self.model_id.clone().unwrap_or_default();
let client = self.build().await?;
let result = client
.model
.as_ref()
.as_ref()
.generate(prompt, GenerateOptions::default())
.await
.map_err(|e| {
match &e {
AiError::ProviderError { message, .. }
if message.to_lowercase().contains("audio")
|| message.to_lowercase().contains("unsupported")
|| message.to_lowercase().contains("media type") =>
{
AiError::UnsupportedCapability {
capability: "audio_transcription".to_string(),
provider: model_id_hint.clone(),
}
}
_ => e,
}
})?;
result.text.ok_or_else(|| AiError::UnsupportedCapability {
capability: "audio_transcription".to_string(),
provider: model_id_hint,
})
}
async fn build(self) -> AiResult<Client> {
let model_id = self.model_id.ok_or_else(|| AiError::ProviderError {
provider: "rs_ai".to_string(),
status: None,
message: "Model ID not set. Use .model() to specify a model.".to_string(),
})?;
let api_key = self.api_key.ok_or_else(|| AiError::AuthError {
message: "API key not set. Use .api_key() to specify credentials.".to_string(),
})?;
let cf_base = self.cf_gateway.as_deref().map(|gw| {
if gw.starts_with("https://") || gw.starts_with("http://") {
gw.trim_end_matches('/').to_string()
} else {
format!(
"https://gateway.ai.cloudflare.com/v1/{}",
gw.trim_end_matches('/')
)
}
});
let model: Box<dyn LanguageModel> = match self.provider_type {
ProviderType::Claude => {
let mut provider = ClaudeProvider::new(api_key);
if let Some(gw) = &cf_base {
provider = provider.with_base_url(format!("{}/anthropic", gw));
}
let mut m = provider.model(&model_id);
if let Some(cache_cfg) = &self.cache_config {
m.set_cache(cache_cfg.clone());
}
Box::new(m)
}
ProviderType::ChatGpt => {
if let Some(gw) = &cf_base {
let config = OpenAiCompatibleConfig::new(format!("{}/openai", gw), api_key);
let provider = OpenAiCompatibleProvider::new(config, "chatgpt", "ChatGPT");
provider.language_model(&model_id)
} else {
let provider = ChatGptProvider::new(api_key);
let mut m = provider.model(&model_id);
if let Some(cache_cfg) = &self.cache_config {
m.set_cache(cache_cfg.clone());
}
Box::new(m)
}
}
ProviderType::Gemini => {
let provider = GeminiProvider::new(api_key);
if let Some(gw) = &cf_base {
let mut m = provider.model_with_base_url(
&model_id,
format!("{}/google-ai-studio/v1/models", gw),
);
if let Some(cache_cfg) = &self.cache_config {
m.set_cache(cache_cfg.clone());
}
Box::new(m)
} else {
let mut m = provider.model(&model_id);
if let Some(cache_cfg) = &self.cache_config {
m.set_cache(cache_cfg.clone());
}
Box::new(m)
}
}
ProviderType::Xai => {
if let Some(gw) = &cf_base {
let config = OpenAiCompatibleConfig::new(format!("{}/grok", gw), api_key);
let provider = OpenAiCompatibleProvider::new(config, "xai", "xAI Grok");
provider.language_model(&model_id)
} else {
let provider = XaiProvider::new(api_key);
let mut m = provider.model(&model_id);
if let Some(cache_cfg) = &self.cache_config {
m.set_cache(cache_cfg.clone());
}
Box::new(m)
}
}
ProviderType::Cloudflare { account_id } => {
let provider = CloudflareProvider::new(account_id, api_key);
Box::new(provider.model(&model_id))
}
ProviderType::Compatible { base_url } => {
let config = OpenAiCompatibleConfig::new(&base_url, &api_key);
let provider = OpenAiCompatibleProvider::new(config, "custom", "OpenAI Compatible");
provider.language_model(&model_id)
}
};
Ok(Client {
model: std::sync::Arc::new(model),
})
}
}
fn image_data_from_url_or_path(url_or_path: &str) -> AiResult<ImageData> {
if url_or_path.starts_with("http://") || url_or_path.starts_with("https://") {
Ok(ImageData::Url {
url: url_or_path.to_string(),
detail: None,
})
} else {
let bytes = std::fs::read(url_or_path).map_err(|e| AiError::Transport {
message: format!("Failed to read image file `{url_or_path}`: {e}"),
source: None,
})?;
let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes);
let media_type = if url_or_path.ends_with(".png") {
"image/png"
} else if url_or_path.ends_with(".gif") {
"image/gif"
} else if url_or_path.ends_with(".webp") {
"image/webp"
} else {
"image/jpeg"
};
Ok(ImageData::Base64 {
media_type: media_type.to_string(),
data: encoded,
})
}
}
fn build_vision_message(text: String, images: Vec<String>) -> AiResult<Message> {
let mut content = vec![ContentPart::Text { text }];
for img in images {
let data = image_data_from_url_or_path(&img)?;
content.push(ContentPart::Image { data });
}
Ok(Message {
role: rs_ai_core::Role::User,
content,
name: None,
metadata: std::collections::HashMap::new(),
})
}
#[derive(Clone)]
pub struct Client {
model: std::sync::Arc<Box<dyn LanguageModel>>,
}
impl Client {
pub async fn generate(&self, prompt: impl Into<String>) -> AiResult<String> {
self.generate_prompt(Prompt::Text(prompt.into())).await
}
pub async fn generate_prompt(&self, prompt: Prompt) -> AiResult<String> {
let result = self
.model
.as_ref()
.as_ref()
.generate(prompt, GenerateOptions::default())
.await?;
result.text.ok_or_else(|| AiError::ProviderError {
provider: self.model.as_ref().as_ref().provider_id().to_string(),
status: None,
message: "No text in response (model returned only tool calls)".to_string(),
})
}
pub async fn generate_with_images(
&self,
prompt: impl Into<String>,
images: Vec<String>,
) -> AiResult<String> {
let message = build_vision_message(prompt.into(), images)?;
let result = self
.model
.as_ref()
.as_ref()
.generate(Prompt::Messages(vec![message]), GenerateOptions::default())
.await?;
result.text.ok_or_else(|| AiError::ProviderError {
provider: self.model.as_ref().as_ref().provider_id().to_string(),
status: None,
message: "No text in response (model returned only tool calls)".to_string(),
})
}
pub async fn stream(
&self,
prompt: impl Into<String>,
) -> AiResult<BoxStream<'static, AiResult<StreamEvent>>> {
self.stream_prompt(Prompt::Text(prompt.into())).await
}
pub async fn stream_prompt(
&self,
prompt: Prompt,
) -> AiResult<BoxStream<'static, AiResult<StreamEvent>>> {
self.model
.as_ref()
.as_ref()
.stream(prompt, GenerateOptions::default())
.await
}
pub async fn stream_with_images(
&self,
prompt: impl Into<String>,
images: Vec<String>,
) -> AiResult<BoxStream<'static, AiResult<StreamEvent>>> {
let message = build_vision_message(prompt.into(), images)?;
self.model
.as_ref()
.as_ref()
.stream(Prompt::Messages(vec![message]), GenerateOptions::default())
.await
}
pub fn model(&self) -> &dyn LanguageModel {
self.model.as_ref().as_ref()
}
}
pub fn claude() -> ClientBuilder {
ClientBuilder {
provider_type: ProviderType::Claude,
api_key: None,
model_id: None,
images: Vec::new(),
cf_gateway: None,
cache_config: None,
}
}
pub fn chatgpt() -> ClientBuilder {
ClientBuilder {
provider_type: ProviderType::ChatGpt,
api_key: None,
model_id: None,
images: Vec::new(),
cf_gateway: None,
cache_config: None,
}
}
pub fn gemini() -> ClientBuilder {
ClientBuilder {
provider_type: ProviderType::Gemini,
api_key: None,
model_id: None,
images: Vec::new(),
cf_gateway: None,
cache_config: None,
}
}
pub fn xai() -> ClientBuilder {
ClientBuilder {
provider_type: ProviderType::Xai,
api_key: None,
model_id: None,
images: Vec::new(),
cf_gateway: None,
cache_config: None,
}
}
pub fn cloudflare(account_id: impl Into<String>) -> ClientBuilder {
ClientBuilder {
provider_type: ProviderType::Cloudflare {
account_id: account_id.into(),
},
api_key: None,
model_id: None,
images: Vec::new(),
cf_gateway: None,
cache_config: None,
}
}
pub fn compatible(base_url: impl Into<String>) -> ClientBuilder {
ClientBuilder {
provider_type: ProviderType::Compatible {
base_url: base_url.into(),
},
api_key: None,
model_id: None,
images: Vec::new(),
cf_gateway: None,
cache_config: None,
}
}