use std::fmt;
use std::future::Future;
use std::num::NonZeroUsize;
use std::sync::Arc;
use serde::{Deserialize, Deserializer, Serialize};
use crate::KitError;
use crate::batch::{BatchJob, BatchSubmitRequest};
use crate::embedding::{EmbedRequest, EmbedResponse};
use crate::error::Operation;
use crate::error::ValidationViolation;
use crate::generation::{
GenerateEventStream, GenerateRequest, GenerateResponse, GenerationContinuation,
};
use crate::provider::ProviderRuntime;
use crate::providers::{
anthropic::AnthropicModel, gemini::GeminiModel, ollama::OllamaModel, openai::OpenAiModel,
};
use crate::session::{Session, SessionConfig, SessionInner};
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
#[serde(transparent)]
pub struct ModelId(String);
impl ModelId {
pub fn new(value: impl Into<String>) -> Result<Self, KitError> {
let value = value.into();
if value.is_empty()
|| value.len() > 256
|| value.trim() != value
|| value.chars().any(char::is_control)
{
return Err(KitError::invalid_input(
ValidationViolation::InvalidFormat {
field: "model_id",
rule: "1..=256 unpadded UTF-8 bytes without control characters",
},
"model id must contain 1..=256 unpadded UTF-8 bytes without control characters",
));
}
Ok(Self(value))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Display for ModelId {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for ModelId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Self::new(String::deserialize(deserializer)?).map_err(serde::de::Error::custom)
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ModelListRequest {
limit: Option<NonZeroUsize>,
cursor: Option<String>,
}
impl ModelListRequest {
#[must_use]
pub fn with_limit(mut self, limit: NonZeroUsize) -> Self {
self.limit = Some(limit);
self
}
pub fn with_cursor(mut self, cursor: impl Into<String>) -> Result<Self, KitError> {
let cursor = cursor.into();
if cursor.trim().is_empty() {
return Err(KitError::invalid_input(
ValidationViolation::Empty {
field: "models.cursor",
},
"model cursor must not be empty",
));
}
self.cursor = Some(cursor);
Ok(self)
}
#[must_use]
pub const fn limit(&self) -> Option<NonZeroUsize> {
self.limit
}
#[must_use]
pub fn cursor(&self) -> Option<&str> {
self.cursor.as_deref()
}
#[must_use]
pub fn into_parts(self) -> (Option<NonZeroUsize>, Option<String>) {
(self.limit, self.cursor)
}
}
impl<'de> Deserialize<'de> for ModelListRequest {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Wire {
limit: Option<NonZeroUsize>,
cursor: Option<String>,
}
let wire = Wire::deserialize(deserializer)?;
let mut request = Self {
limit: wire.limit,
cursor: None,
};
if let Some(cursor) = wire.cursor {
request = request
.with_cursor(cursor)
.map_err(serde::de::Error::custom)?;
}
Ok(request)
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ModelSummary {
pub id: ModelId,
pub display_name: Option<String>,
pub description: Option<String>,
pub context_window_tokens: Option<u64>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ModelPage {
pub models: Vec<ModelSummary>,
pub next_cursor: Option<String>,
}
pub trait ModelBackend: Send + Sync + 'static {
type Continuation: Clone + Send + Sync + 'static;
type Session: crate::session::SessionBackend;
fn model_id(&self) -> &ModelId;
fn generate<'a>(
&'a self,
request: &'a GenerateRequest<Self::Continuation>,
) -> impl Future<Output = Result<GenerateResponse<Self::Continuation>, KitError>> + Send + 'a;
fn generate_stream<'a>(
&'a self,
request: &'a GenerateRequest<Self::Continuation>,
) -> impl Future<Output = Result<GenerateEventStream<Self::Continuation>, KitError>> + Send + 'a;
fn embed<'a>(
&'a self,
_request: &'a EmbedRequest,
) -> impl Future<Output = Result<EmbedResponse, KitError>> + Send + 'a {
async {
Err(KitError::unsupported(
ValidationViolation::Unsupported {
field: "embedding",
rule: "operation is not implemented",
},
"model embeddings",
)
.at_operation(Operation::Embed))
}
}
fn submit_batch<'a>(
&'a self,
_request: &'a BatchSubmitRequest<Self::Continuation>,
) -> impl Future<Output = Result<BatchJob, KitError>> + Send + 'a {
async {
Err(KitError::unsupported(
ValidationViolation::Unsupported {
field: "batch.submit",
rule: "operation is not implemented",
},
"model batch submission",
)
.at_operation(Operation::SubmitBatch))
}
}
fn open_session(
&self,
_config: SessionConfig,
) -> impl Future<Output = Result<Self::Session, KitError>> + Send + '_ {
async {
Err(KitError::unsupported(
ValidationViolation::Unsupported {
field: "session",
rule: "operation is not implemented",
},
"model interactive sessions",
)
.at_operation(Operation::OpenSession))
}
}
}
#[derive(Clone)]
pub(crate) enum ModelInner {
OpenAi(OpenAiModel),
Anthropic(AnthropicModel),
Gemini(GeminiModel),
Ollama(OllamaModel),
}
#[derive(Clone)]
pub struct Model {
inner: ModelInner,
provider: Arc<ProviderRuntime>,
}
impl Model {
pub(crate) fn new(inner: ModelInner, provider: Arc<ProviderRuntime>) -> Self {
Self { inner, provider }
}
}
impl ModelBackend for ModelInner {
type Continuation = GenerationContinuation;
type Session = SessionInner;
fn model_id(&self) -> &ModelId {
match self {
Self::OpenAi(model) => model.model_id(),
Self::Anthropic(model) => model.model_id(),
Self::Gemini(model) => model.model_id(),
Self::Ollama(model) => model.model_id(),
}
}
async fn generate(
&self,
request: &GenerateRequest<GenerationContinuation>,
) -> Result<GenerateResponse<GenerationContinuation>, KitError> {
match self {
Self::OpenAi(model) => model.generate(request).await,
Self::Anthropic(model) => model.generate(request).await,
Self::Gemini(model) => model.generate(request).await,
Self::Ollama(model) => model.generate(request).await,
}
}
async fn generate_stream(
&self,
request: &GenerateRequest<GenerationContinuation>,
) -> Result<GenerateEventStream<GenerationContinuation>, KitError> {
match self {
Self::OpenAi(model) => model.generate_stream(request).await,
Self::Anthropic(model) => model.generate_stream(request).await,
Self::Gemini(model) => model.generate_stream(request).await,
Self::Ollama(model) => model.generate_stream(request).await,
}
}
async fn embed(&self, request: &EmbedRequest) -> Result<EmbedResponse, KitError> {
match self {
Self::OpenAi(model) => model.embed(request).await,
Self::Anthropic(model) => model.embed(request).await,
Self::Gemini(model) => model.embed(request).await,
Self::Ollama(model) => model.embed(request).await,
}
}
async fn submit_batch(
&self,
request: &BatchSubmitRequest<GenerationContinuation>,
) -> Result<BatchJob, KitError> {
match self {
Self::OpenAi(model) => model.submit_batch(request).await,
Self::Anthropic(model) => model.submit_batch(request).await,
Self::Gemini(model) => model.submit_batch(request).await,
Self::Ollama(model) => model.submit_batch(request).await,
}
}
async fn open_session(&self, config: SessionConfig) -> Result<SessionInner, KitError> {
match self {
Self::OpenAi(model) => model.open_session(config).await.map(SessionInner::OpenAi),
Self::Anthropic(model) => model.open_session(config).await.map(|never| match never {}),
Self::Gemini(model) => model.open_session(config).await.map(|never| match never {}),
Self::Ollama(model) => model.open_session(config).await.map(|never| match never {}),
}
}
}
impl ModelBackend for Model {
type Continuation = GenerationContinuation;
type Session = Session;
fn model_id(&self) -> &ModelId {
self.inner.model_id()
}
async fn generate(
&self,
request: &GenerateRequest<GenerationContinuation>,
) -> Result<GenerateResponse<GenerationContinuation>, KitError> {
let _admission = self.provider.admit(Operation::Generate)?;
self.inner
.generate(request)
.await
.map_err(|error| error.at_operation(Operation::Generate))
}
async fn generate_stream(
&self,
request: &GenerateRequest<GenerationContinuation>,
) -> Result<GenerateEventStream<GenerationContinuation>, KitError> {
let admission = self.provider.admit(Operation::GenerateStream)?;
let stream = self
.inner
.generate_stream(request)
.await
.map_err(|error| error.at_operation(Operation::GenerateStream))?;
Ok(admission.bind(stream))
}
async fn embed(&self, request: &EmbedRequest) -> Result<EmbedResponse, KitError> {
let _admission = self.provider.admit(Operation::Embed)?;
self.inner
.embed(request)
.await
.map_err(|error| error.at_operation(Operation::Embed))
}
async fn submit_batch(
&self,
request: &BatchSubmitRequest<GenerationContinuation>,
) -> Result<BatchJob, KitError> {
let admission = self.provider.admit(Operation::SubmitBatch)?;
if let (ModelInner::OpenAi(model), BatchSubmitRequest::Inline { .. }) =
(&self.inner, request)
{
let submission = model
.prepare_owned_batch(request)
.map_err(|error| error.at_operation(Operation::SubmitBatch))?;
return self.provider.run_owned_batch(submission, admission).await;
}
let _admission = admission;
self.inner
.submit_batch(request)
.await
.map_err(|error| error.at_operation(Operation::SubmitBatch))
}
async fn open_session(&self, config: SessionConfig) -> Result<Session, KitError> {
let _admission = self.provider.admit(Operation::OpenSession)?;
let session = self
.inner
.open_session(config)
.await
.map_err(|error| error.at_operation(Operation::OpenSession))?;
Ok(Session::new(session, Arc::clone(&self.provider)))
}
}
impl fmt::Debug for Model {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Model")
.field("model_id", self.model_id())
.finish()
}
}