use chrono::{DateTime, Utc};
use nutype::nutype;
use serde::{Deserialize, Serialize};
use crate::domain::llm::{LlmProvider, ModelVersion};
use crate::domain::types::{ChangeReason, RequestCount};
#[nutype(derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize))]
pub struct VersionChangeId(uuid::Uuid);
impl VersionChangeId {
pub fn generate() -> Self {
Self::new(uuid::Uuid::now_v7())
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TrackedVersion {
pub version: ModelVersion,
pub first_seen: DateTime<Utc>,
pub last_seen: DateTime<Utc>,
pub request_count: RequestCount,
pub is_active: bool,
}
impl TrackedVersion {
pub fn new(version: ModelVersion) -> Self {
let now = Utc::now();
Self {
version,
first_seen: now,
last_seen: now,
request_count: RequestCount::new(1),
is_active: true,
}
}
pub fn record_usage(&mut self) {
self.last_seen = Utc::now();
self.request_count = self.request_count.increment();
}
pub fn deactivate(&mut self) {
self.is_active = false;
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum VersionComparison {
Same,
Changed {
from_provider: LlmProvider,
from_model_id: String,
to_provider: LlmProvider,
to_model_id: String,
},
}
impl ModelVersion {
pub fn compare(&self, other: &ModelVersion) -> VersionComparison {
if self.provider == other.provider && self.model_id == other.model_id {
VersionComparison::Same
} else {
VersionComparison::Changed {
from_provider: self.provider.clone(),
from_model_id: self.model_id.as_ref().to_string(),
to_provider: other.provider.clone(),
to_model_id: other.model_id.as_ref().to_string(),
}
}
}
pub fn to_version_string(&self) -> String {
format!("{}/{}", self.provider.as_str(), self.model_id.as_ref())
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct VersionTestConfig {
pub target_version: Option<ModelVersion>,
pub compare_mode: bool,
pub baseline_version: Option<ModelVersion>,
pub mode: TestExecutionMode,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum TestExecutionMode {
Original,
Target,
Comparison,
AllVersions,
}
impl Default for VersionTestConfig {
fn default() -> Self {
Self {
target_version: None,
compare_mode: false,
baseline_version: None,
mode: TestExecutionMode::Original,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct VersionChangeEvent {
pub id: VersionChangeId,
pub session_id: crate::domain::SessionId,
pub from_version: ModelVersion,
pub to_version: ModelVersion,
pub change_type: VersionComparison,
pub occurred_at: DateTime<Utc>,
pub reason: Option<ChangeReason>,
}
impl VersionChangeEvent {
pub fn new(
session_id: crate::domain::SessionId,
from_version: ModelVersion,
to_version: ModelVersion,
reason: Option<ChangeReason>,
) -> Self {
let change_type = from_version.compare(&to_version);
Self {
id: VersionChangeId::generate(),
session_id,
from_version,
to_version,
change_type,
occurred_at: Utc::now(),
reason,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::types::ModelId;
use crate::domain::SessionId;
#[test]
fn test_version_comparison_same() {
let v1 = ModelVersion {
provider: LlmProvider::OpenAI,
model_id: ModelId::try_new("gpt-4-turbo-2024-01".to_string()).unwrap(),
};
let v2 = v1.clone();
assert_eq!(v1.compare(&v2), VersionComparison::Same);
}
#[test]
fn test_version_comparison_changed() {
let v1 = ModelVersion {
provider: LlmProvider::OpenAI,
model_id: ModelId::try_new("gpt-3.5-turbo".to_string()).unwrap(),
};
let v2 = ModelVersion {
provider: LlmProvider::OpenAI,
model_id: ModelId::try_new("gpt-4-turbo-2024-01".to_string()).unwrap(),
};
assert_eq!(
v1.compare(&v2),
VersionComparison::Changed {
from_provider: LlmProvider::OpenAI,
from_model_id: "gpt-3.5-turbo".to_string(),
to_provider: LlmProvider::OpenAI,
to_model_id: "gpt-4-turbo-2024-01".to_string(),
}
);
}
#[test]
fn test_version_comparison_provider_changed() {
let v1 = ModelVersion {
provider: LlmProvider::OpenAI,
model_id: ModelId::try_new("gpt-4-turbo".to_string()).unwrap(),
};
let v2 = ModelVersion {
provider: LlmProvider::Anthropic,
model_id: ModelId::try_new("claude-3-opus-20240229".to_string()).unwrap(),
};
assert_eq!(
v1.compare(&v2),
VersionComparison::Changed {
from_provider: LlmProvider::OpenAI,
from_model_id: "gpt-4-turbo".to_string(),
to_provider: LlmProvider::Anthropic,
to_model_id: "claude-3-opus-20240229".to_string(),
}
);
}
#[test]
fn test_tracked_version_usage() {
let version = ModelVersion {
provider: LlmProvider::OpenAI,
model_id: ModelId::try_new("gpt-4-turbo-2024-01".to_string()).unwrap(),
};
let mut tracked = TrackedVersion::new(version);
assert_eq!(tracked.request_count, RequestCount::new(1));
assert!(tracked.is_active);
tracked.record_usage();
assert_eq!(tracked.request_count, RequestCount::new(2));
tracked.deactivate();
assert!(!tracked.is_active);
}
#[test]
fn test_version_string_formatting() {
let version = ModelVersion {
provider: LlmProvider::OpenAI,
model_id: ModelId::try_new("gpt-4-turbo-2024-01".to_string()).unwrap(),
};
assert_eq!(version.to_version_string(), "openai/gpt-4-turbo-2024-01");
let version_anthropic = ModelVersion {
provider: LlmProvider::Anthropic,
model_id: ModelId::try_new("claude-3-opus-20240229".to_string()).unwrap(),
};
assert_eq!(
version_anthropic.to_version_string(),
"anthropic/claude-3-opus-20240229"
);
}
#[test]
fn test_version_change_event_creation() {
let session_id = SessionId::generate();
let v1 = ModelVersion {
provider: LlmProvider::OpenAI,
model_id: ModelId::try_new("gpt-3.5-turbo".to_string()).unwrap(),
};
let v2 = ModelVersion {
provider: LlmProvider::OpenAI,
model_id: ModelId::try_new("gpt-4-turbo-2024-01".to_string()).unwrap(),
};
let event = VersionChangeEvent::new(
session_id,
v1.clone(),
v2.clone(),
Some(ChangeReason::try_new("Upgrade for better performance".to_string()).unwrap()),
);
assert_eq!(event.from_version, v1);
assert_eq!(event.to_version, v2);
assert_eq!(
event.change_type,
VersionComparison::Changed {
from_provider: LlmProvider::OpenAI,
from_model_id: "gpt-3.5-turbo".to_string(),
to_provider: LlmProvider::OpenAI,
to_model_id: "gpt-4-turbo-2024-01".to_string(),
}
);
assert_eq!(
event.reason.as_ref().map(|r| r.as_ref()),
Some("Upgrade for better performance")
);
}
#[test]
fn test_version_test_config_default() {
let config = VersionTestConfig::default();
assert_eq!(config.mode, TestExecutionMode::Original);
assert!(!config.compare_mode);
assert!(config.target_version.is_none());
assert!(config.baseline_version.is_none());
}
}