Expand description
§Octolib - Self-sufficient AI Provider Library
A comprehensive library for interacting with multiple AI providers through a unified interface.
§Features
- Multi-provider support: OpenAI, Anthropic, OpenRouter, Cerebras, Ollama, Local (LM Studio, LocalAI, Jan, vLLM), Google Vertex AI, Amazon Bedrock, Cloudflare Workers AI, DeepSeek, Moonshot AI (Kimi), Z.ai, CLI proxies (codex, claude, gemini, others)
- Unified interface: Single trait for all providers with consistent API
- Model validation: Strict
provider:modelformat validation - Structured output: JSON and JSON Schema support for OpenAI, OpenRouter, DeepSeek, and Z.ai
- Cost tracking: Automatic token usage and cost calculation
- Vision support: Image attachment support for compatible models
- Caching support: Automatic detection of caching-capable models
- Retry logic: Exponential backoff with smart rate limit handling
- Embeddings: Multi-provider embedding support (Jina, Voyage, Google, OpenAI, FastEmbed, HuggingFace)
- Reranking: Document relevance scoring with cross-encoder models (Voyage AI)
- Self-sufficient: No external dependencies on application-specific types
- CLI provider:
cli:<backend>/<model>proxies CLIs; tool calling/MCP is not used or controllable (prompt-only)
§Usage
§Basic Chat Completion
use octolib::llm::{ProviderFactory, ChatCompletionParams, Message};
// This example shows basic usage but requires API keys to run
async fn example() -> anyhow::Result<()> {
// Parse model and get provider
let (provider, model) = ProviderFactory::get_provider_for_model("openai:gpt-4o")?;
// Create messages
let messages = vec![
Message::user("Hello, how are you?"),
];
// Create completion parameters
let params = ChatCompletionParams::new(&messages, &model, 0.7, 1.0, 50, 1000);
// Get completion (requires OPENAI_API_KEY environment variable)
let response = provider.chat_completion(params).await?;
println!("Response: {}", response.content);
Ok(())
}§Structured Output
use octolib::llm::{ProviderFactory, ChatCompletionParams, Message, StructuredOutputRequest};
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug)]
struct PersonInfo {
name: String,
age: u32,
skills: Vec<String>,
}
async fn structured_example() -> anyhow::Result<()> {
// Works with OpenAI, OpenRouter, and DeepSeek
let (provider, model) = ProviderFactory::get_provider_for_model("deepseek:deepseek-chat")?;
// Check if provider supports structured output
if !provider.supports_structured_output(&model) {
return Err(anyhow::anyhow!("Provider does not support structured output"));
}
let messages = vec![
Message::user("Tell me about a software engineer in JSON format"),
];
// Request structured JSON output
let structured_request = StructuredOutputRequest::json();
let params = ChatCompletionParams::new(&messages, &model, 0.7, 1.0, 50, 1000)
.with_structured_output(structured_request);
let response = provider.chat_completion(params).await?;
if let Some(structured) = response.structured_output {
let person: PersonInfo = serde_json::from_value(structured)?;
println!("Person: {:?}", person);
}
Ok(())
}Re-exports§
pub use embedding::count_tokens;pub use embedding::create_embedding_provider_from_parts;pub use embedding::generate_embeddings;pub use embedding::generate_embeddings_batch;pub use embedding::split_texts_into_token_limited_batches;pub use embedding::truncate_output;pub use embedding::EmbeddingProvider;pub use embedding::EmbeddingProviderType;pub use embedding::InputType;pub use embedding::LocalEmbeddingProvider;pub use embedding::OctoHubEmbeddingProvider;pub use errors::ConfigError;pub use errors::ConfigResult;pub use errors::MessageError;pub use errors::MessageResult;pub use errors::ProviderError;pub use errors::ProviderResult;pub use errors::StructuredOutputError;pub use errors::StructuredOutputResult;pub use errors::ToolCallError;pub use errors::ToolCallResult;pub use llm::AiProvider;pub use llm::AmazonBedrockProvider;pub use llm::AnthropicProvider;pub use llm::CacheConfig;pub use llm::CacheTTL;pub use llm::CacheType;pub use llm::CerebrasProvider;pub use llm::ChatCompletionParams;pub use llm::CloudflareWorkersAiProvider;pub use llm::DeepSeekProvider;pub use llm::EffectiveSamplingParams;pub use llm::FireworksProvider;pub use llm::FunctionDefinition;pub use llm::GenericToolCall;pub use llm::GoogleVertexProvider;pub use llm::ImageAttachment;pub use llm::ImageData;pub use llm::LocalProvider;pub use llm::Message;pub use llm::MessageBuilder;pub use llm::MinimaxProvider;pub use llm::ModelLimits;pub use llm::MoonshotProvider;pub use llm::OllamaProvider;pub use llm::OpenAiProvider;pub use llm::OpenRouterProvider;pub use llm::OutputFormat;pub use llm::ProviderExchange;pub use llm::ProviderFactory;pub use llm::ProviderResponse;pub use llm::ProviderStrategy;pub use llm::ProviderToolCalls;pub use llm::ReasoningEffort;pub use llm::ResponseMode;pub use llm::SamplingSupport;pub use llm::SourceType;pub use llm::StrategyFactory;pub use llm::StructuredOutputRequest;pub use llm::ThinkingBlock;pub use llm::TogetherProvider;pub use llm::TokenUsage;pub use llm::ToolCall;pub use llm::ToolResult;pub use llm::VideoAttachment;pub use llm::VideoData;pub use llm::ZaiProvider;pub use reranker::create_rerank_provider_from_parts;pub use reranker::parse_provider_model as parse_rerank_provider_model;pub use reranker::rerank;pub use reranker::rerank_with_truncation;pub use reranker::RerankProvider;pub use reranker::RerankProviderType;pub use reranker::RerankResponse;pub use reranker::RerankResult;