#[cfg(feature = "http")]
use axum::extract::FromRef;
use std::sync::Arc;
#[cfg(feature = "http")]
use tokio::sync::RwLock;
#[cfg(any(feature = "http", feature = "mcp", feature = "cli"))]
pub mod api;
pub mod audit;
#[cfg(feature = "auth")]
pub mod auth;
pub mod config;
#[cfg(feature = "db")]
pub mod db;
pub mod doctor;
pub mod domain;
pub mod engine;
pub mod i18n;
pub mod library;
pub mod metrics;
pub mod pipeline;
pub mod rate_limit;
pub mod registry;
pub mod security;
pub mod service;
pub mod utils;
pub mod logger;
pub use crate::pipeline::PriorityConfig;
pub(crate) mod cache;
pub(crate) mod device;
pub mod error;
pub(crate) mod model;
pub(crate) mod monitor;
pub(crate) mod text;
pub use config::VecboostConfig;
#[cfg(feature = "db")]
pub use config::app::DatabaseConfig;
pub use config::app::{AuthConfig, CsrfConfig, RateLimitConfig, RerankConfig, ServerConfig};
pub use config::model::ModelConfig;
pub use domain::{
EmbedRequest, EmbedResponse, RerankRequest, RerankResponse, SimilarityRequest,
SimilarityResponse,
};
pub use error::VecboostError;
pub use library::{LibraryConfig, VecBoostLibrary, VecBoostModuleBuilder};
pub use service::embedding::EmbeddingService;
pub use service::rerank::RerankService;
pub use utils::SimilarityMetric;
pub use utils::vector::{TaskType, information_retention_rate, recommended_dimension};
pub use device::memory_paging::{PagingConfig, PagingStats, WeightPagingManager};
pub mod model_management {
pub use crate::model::heat::DEFAULT_HEAT_PATH;
pub use crate::model::loader::LocalModelLoader;
pub use crate::model::manager::ModelManager;
}
pub mod thread_tune {
pub use crate::device::thread_tune::{
detect_physical_cores, parse_lscpu_sockets, parse_thread_siblings_lists,
resolve_worker_threads,
};
}
pub mod planner {
pub use crate::device::planner::{
Bottleneck, HardwarePlan, PlanOverride, Probes, apply_plan, plan,
};
}
pub use cache::{ComparisonMode, SemanticCache, SemanticCacheConfig, SemanticCacheStats};
#[cfg(feature = "grpc")]
pub use sdforge;
#[derive(Clone)]
pub struct VecboostState {
pub(crate) kit: Arc<trait_kit::AsyncKit<trait_kit::AsyncReady>>,
}
impl VecboostState {
pub fn new(kit: Arc<trait_kit::AsyncKit<trait_kit::AsyncReady>>) -> Self {
Self { kit }
}
pub fn kit(&self) -> &Arc<trait_kit::AsyncKit<trait_kit::AsyncReady>> {
&self.kit
}
}
#[cfg(feature = "http")]
macro_rules! impl_from_ref_direct {
($target:ty, $module:ty, $msg:literal) => {
#[cfg(feature = "http")]
impl FromRef<VecboostState> for $target {
fn from_ref(state: &VecboostState) -> Self {
state.kit.require::<$module>().expect($msg)
}
}
};
}
#[cfg(feature = "http")]
macro_rules! impl_from_ref_option {
($target:ty, $module:ty, $key:literal, $msg:literal) => {
#[cfg(feature = "http")]
impl FromRef<VecboostState> for $target {
fn from_ref(state: &VecboostState) -> Self {
state
.kit
.require::<$module>()
.and_then(|opt| {
opt.ok_or_else(|| trait_kit::TraitKitError::MissingCapability {
key: $key.to_string(),
})
})
.expect($msg)
}
}
};
}
#[cfg(feature = "http")]
impl_from_ref_direct!(
Arc<RwLock<EmbeddingService>>,
registry::EmbeddingModule,
"EmbeddingService capability not registered in kit"
);
#[cfg(feature = "http")]
impl_from_ref_direct!(
Arc<RwLock<RerankService>>,
registry::RerankModule,
"RerankService capability not registered in kit"
);
#[cfg(feature = "http")]
impl_from_ref_direct!(
Arc<rate_limit::LimiteronAdapter>,
registry::RateLimitModule,
"RateLimitModule capability not registered in kit"
);
#[cfg(feature = "http")]
impl_from_ref_direct!(
Option<Arc<audit::AuditLogger>>,
registry::AuditModule,
"AuditModule capability not registered in kit"
);
#[cfg(all(feature = "http", feature = "auth"))]
impl_from_ref_option!(
Arc<auth::GarrisonCsrfConfig>,
registry::CsrfConfigModule,
"csrf_config (auth disabled at runtime)",
"GarrisonCsrfConfig capability not available"
);
#[cfg(feature = "http")]
impl_from_ref_option!(
Arc<metrics::InferenceCollector>,
registry::MetricsCollectorModule,
"metrics_collector (not configured)",
"InferenceCollector capability not available"
);
#[cfg(feature = "http")]
impl_from_ref_option!(
Arc<metrics::PrometheusCollector>,
registry::PrometheusCollectorModule,
"prometheus_collector (not configured)",
"PrometheusCollector capability not available"
);
#[cfg(all(feature = "http", feature = "auth"))]
impl FromRef<VecboostState> for config::app::AuthConfig {
fn from_ref(state: &VecboostState) -> Self {
state
.kit
.config::<config::app::AuthConfig>()
.unwrap_or_default()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "http")]
use crate::config::model::Precision;
#[cfg(feature = "http")]
use crate::engine::InferenceEngine;
use crate::logger::LoggerModule;
#[cfg(feature = "http")]
use crate::pipeline::{PriorityConfig, WorkerConfig};
#[cfg(feature = "http")]
use crate::registry::PrometheusCollectorModule;
#[cfg(feature = "http")]
use crate::registry::{
AuditModule, AuthEnabled, CacheConfig, CacheModule, DbConfig, DbModule, EmbeddingModule,
IpWhitelistModule, MetricsCollectorModule, PipelineEnabled, PipelineQueueModule,
PriorityCalculatorModule, RateLimitEnabled, RateLimitModule, RerankModule,
ResponseChannelModule, WorkerManagerModule,
};
#[cfg(feature = "auth")]
use crate::registry::{AuthModule, CsrfConfigModule};
#[cfg(feature = "http")]
use async_trait::async_trait;
#[cfg(feature = "http")]
struct MockEngine;
#[cfg(feature = "http")]
#[async_trait]
impl InferenceEngine for MockEngine {
fn embed(&self, _text: &str) -> Result<Vec<f32>, VecboostError> {
Ok(vec![0.0; 384])
}
fn embed_batch(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, VecboostError> {
Ok(texts.iter().map(|_| vec![0.0; 384]).collect())
}
fn precision(&self) -> &Precision {
&Precision::Fp32
}
fn supports_mixed_precision(&self) -> bool {
false
}
async fn try_fallback_to_cpu(
&mut self,
_config: &crate::config::model::ModelConfig,
) -> Result<(), VecboostError> {
Ok(())
}
}
#[cfg(feature = "http")]
async fn make_app_state_with_options(
metrics: Option<Arc<metrics::InferenceCollector>>,
prometheus: Option<Arc<metrics::PrometheusCollector>>,
audit: Option<Arc<audit::AuditLogger>>,
) -> VecboostState {
let engine: Arc<RwLock<dyn InferenceEngine + Send + Sync>> =
Arc::new(RwLock::new(MockEngine));
let service = Arc::new(RwLock::new(EmbeddingService::new(engine.clone(), None)));
let rerank_service = Arc::new(RwLock::new(RerankService::new(engine, None)));
let rate_limiter = Arc::new(rate_limit::LimiteronAdapter::with_defaults().await);
let pipeline_queue = Arc::new(pipeline::PriorityRequestQueue::new(100));
let response_channel = Arc::new(pipeline::ResponseChannel::new());
let priority_calculator =
Arc::new(pipeline::PriorityCalculator::new(PriorityConfig::default()));
let worker_manager = Arc::new(pipeline::WorkerManager::new(
pipeline_queue.clone(),
response_channel.clone(),
WorkerConfig::default(),
service.clone(),
));
let mut kit = trait_kit::AsyncKit::new();
kit.set_config(service.clone());
kit.set_config(rerank_service.clone());
kit.set_config(rate_limiter.clone());
kit.set_config(metrics.clone());
kit.set_config(prometheus.clone());
kit.set_config(audit.clone());
kit.set_config(pipeline_queue.clone());
kit.set_config(response_channel.clone());
kit.set_config(priority_calculator.clone());
kit.set_config(worker_manager.clone());
kit.set_config(Vec::<String>::new());
kit.set_config(AuthEnabled(false));
kit.set_config(RateLimitEnabled(false));
kit.set_config(PipelineEnabled(false));
kit.set_config(CacheConfig {
enabled: false,
size: 0,
});
kit.set_config(DbConfig { enabled: false });
kit.set_config(RerankConfig::default());
let logger_manager = Arc::new(
inklog::LoggerManager::builder()
.level("warn")
.console(false)
.build()
.await
.expect("test logger manager"),
);
kit.set_config(logger_manager);
#[cfg(feature = "auth")]
{
kit.set_config(Option::<Arc<crate::auth::GarrisonHandle>>::None);
kit.set_config(Option::<Arc<crate::auth::GarrisonCsrfConfig>>::None);
}
kit.register::<LoggerModule>().unwrap();
kit.register::<EmbeddingModule>().unwrap();
kit.register::<RerankModule>().unwrap();
kit.register::<RateLimitModule>().unwrap();
kit.register::<CacheModule>().unwrap();
kit.register::<DbModule>().unwrap();
kit.register::<AuditModule>().unwrap();
kit.register::<MetricsCollectorModule>().unwrap();
kit.register::<PrometheusCollectorModule>().unwrap();
kit.register::<IpWhitelistModule>().unwrap();
kit.register::<PipelineQueueModule>().unwrap();
kit.register::<ResponseChannelModule>().unwrap();
kit.register::<PriorityCalculatorModule>().unwrap();
kit.register::<WorkerManagerModule>().unwrap();
#[cfg(feature = "auth")]
{
kit.register::<AuthModule>().unwrap();
kit.register::<CsrfConfigModule>().unwrap();
}
kit.register_lifecycle::<EmbeddingModule>();
kit.register_lifecycle::<RerankModule>();
kit.register_lifecycle::<RateLimitModule>();
kit.register_lifecycle::<AuditModule>();
kit.register_health_check::<EmbeddingModule>();
kit.register_health_check::<RerankModule>();
kit.register_health_check::<RateLimitModule>();
kit.register_health_check::<CacheModule>();
let kit = kit.build().await.expect("Failed to build AsyncKit");
VecboostState { kit: Arc::new(kit) }
}
#[cfg(feature = "http")]
pub(crate) async fn make_app_state() -> VecboostState {
make_app_state_with_options(
Some(Arc::new(metrics::InferenceCollector::new())),
Some(Arc::new(
metrics::PrometheusCollector::new().expect("Failed to create PrometheusCollector"),
)),
None,
)
.await
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_app_state_construction() {
let state = make_app_state().await;
assert!(state.kit.contains::<EmbeddingModule>());
assert!(state.kit.contains::<RerankModule>());
assert!(state.kit.contains::<RateLimitModule>());
assert!(state.kit.contains::<AuditModule>());
assert!(state.kit.contains::<MetricsCollectorModule>());
assert!(state.kit.contains::<PrometheusCollectorModule>());
assert!(state.kit.contains::<IpWhitelistModule>());
assert!(state.kit.contains::<PipelineQueueModule>());
assert!(state.kit.contains::<WorkerManagerModule>());
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_app_state_clone_preserves_kit_arc() {
let state = make_app_state().await;
let cloned = state.clone();
assert!(Arc::ptr_eq(&state.kit, &cloned.kit));
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_app_state_multiple_clones_share_kit() {
let state = make_app_state().await;
let clone1 = state.clone();
let clone2 = state.clone();
let clone3 = state.clone();
assert!(Arc::ptr_eq(&state.kit, &clone1.kit));
assert!(Arc::ptr_eq(&state.kit, &clone2.kit));
assert!(Arc::ptr_eq(&state.kit, &clone3.kit));
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_require_embedding_service() {
let state = make_app_state().await;
let service = state.kit.require::<EmbeddingModule>().unwrap();
let _guard = service.read().await;
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_require_rate_limiter() {
let state = make_app_state().await;
let _limiter = state.kit.require::<RateLimitModule>().unwrap();
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_require_metrics_collector_returns_some() {
let state = make_app_state().await;
let collector = state.kit.require::<MetricsCollectorModule>().unwrap();
assert!(collector.is_some());
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_require_prometheus_collector_returns_some() {
let state = make_app_state().await;
let collector = state.kit.require::<PrometheusCollectorModule>().unwrap();
assert!(collector.is_some());
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_require_audit_logger_returns_none() {
let state = make_app_state().await;
let logger = state.kit.require::<AuditModule>().unwrap();
assert!(logger.is_none());
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_require_ip_whitelist_empty() {
let state = make_app_state().await;
let whitelist = state.kit.require::<IpWhitelistModule>().unwrap();
assert!(whitelist.is_empty());
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_config_bool_flags_default_false() {
let state = make_app_state().await;
let auth_enabled = state
.kit
.config::<AuthEnabled>()
.map(|c| c.0)
.unwrap_or(false);
let rate_limit_enabled = state
.kit
.config::<RateLimitEnabled>()
.map(|c| c.0)
.unwrap_or(false);
let pipeline_enabled = state
.kit
.config::<PipelineEnabled>()
.map(|c| c.0)
.unwrap_or(false);
assert!(!auth_enabled);
assert!(!rate_limit_enabled);
assert!(!pipeline_enabled);
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_require_pipeline_components() {
let state = make_app_state().await;
let _queue = state.kit.require::<PipelineQueueModule>().unwrap();
let _channel = state.kit.require::<ResponseChannelModule>().unwrap();
let _calculator = state.kit.require::<PriorityCalculatorModule>().unwrap();
let _manager = state.kit.require::<WorkerManagerModule>().unwrap();
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_require_cache_and_db_default_false() {
let state = make_app_state().await;
let cache_enabled = state.kit.require::<CacheModule>().unwrap();
let db_enabled = state.kit.require::<DbModule>().unwrap();
assert!(!cache_enabled);
assert!(!db_enabled);
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_require_logger_module() {
let state = make_app_state().await;
assert!(
state.kit.contains::<LoggerModule>(),
"LoggerModule should be registered in kit"
);
let logger: Arc<inklog::LoggerManager> = state
.kit
.require::<LoggerModule>()
.expect("require LoggerModule");
let _ = logger;
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_service() {
let state = make_app_state().await;
let service: Arc<RwLock<EmbeddingService>> = FromRef::from_ref(&state);
let kit_service = state.kit.require::<EmbeddingModule>().unwrap();
assert!(Arc::ptr_eq(&service, &kit_service));
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_rate_limiter() {
let state = make_app_state().await;
let limiter: Arc<rate_limit::LimiteronAdapter> = FromRef::from_ref(&state);
let kit_limiter = state.kit.require::<RateLimitModule>().unwrap();
assert!(Arc::ptr_eq(&limiter, &kit_limiter));
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_metrics_collector() {
let state = make_app_state().await;
let collector: Arc<metrics::InferenceCollector> = FromRef::from_ref(&state);
let kit_collector = state.kit.require::<MetricsCollectorModule>().unwrap();
assert!(Arc::ptr_eq(&collector, kit_collector.as_ref().unwrap()));
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_prometheus_collector() {
let state = make_app_state().await;
let collector: Arc<metrics::PrometheusCollector> = FromRef::from_ref(&state);
let kit_collector = state.kit.require::<PrometheusCollectorModule>().unwrap();
assert!(Arc::ptr_eq(&collector, kit_collector.as_ref().unwrap()));
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_audit_logger_returns_none() {
let state = make_app_state().await;
let logger: Option<Arc<audit::AuditLogger>> = FromRef::from_ref(&state);
assert!(logger.is_none());
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_audit_logger_returns_some() {
let config = audit::AuditConfig {
enabled: false,
..Default::default()
};
let logger = Arc::new(audit::AuditLogger::new(config));
let state = make_app_state_with_options(
Some(Arc::new(metrics::InferenceCollector::new())),
Some(Arc::new(
metrics::PrometheusCollector::new().expect("Failed to create PrometheusCollector"),
)),
Some(logger.clone()),
)
.await;
let extracted: Option<Arc<audit::AuditLogger>> = FromRef::from_ref(&state);
assert!(extracted.is_some());
assert!(Arc::ptr_eq(&extracted.unwrap(), &logger));
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_service_after_clone() {
let state = make_app_state().await;
let cloned = state.clone();
let service: Arc<RwLock<EmbeddingService>> = FromRef::from_ref(&cloned);
let kit_service = state.kit.require::<EmbeddingModule>().unwrap();
assert!(Arc::ptr_eq(&service, &kit_service));
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_rate_limiter_after_clone() {
let state = make_app_state().await;
let cloned = state.clone();
let limiter: Arc<rate_limit::LimiteronAdapter> = FromRef::from_ref(&cloned);
let kit_limiter = state.kit.require::<RateLimitModule>().unwrap();
assert!(Arc::ptr_eq(&limiter, &kit_limiter));
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_metrics_collector_panics_when_none() {
let state = make_app_state_with_options(None, None, None).await;
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _: Arc<metrics::InferenceCollector> = FromRef::from_ref(&state);
}));
assert!(
result.is_err(),
"from_ref should panic when metrics_collector is None"
);
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_from_ref_prometheus_collector_panics_when_none() {
let state = make_app_state_with_options(None, None, None).await;
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _: Arc<metrics::PrometheusCollector> = FromRef::from_ref(&state);
}));
assert!(
result.is_err(),
"from_ref should panic when prometheus_collector is None"
);
}
#[test]
fn test_vecboost_state_is_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<VecboostState>();
}
#[test]
fn test_vecboost_state_is_clone() {
fn assert_clone<T: Clone>() {}
assert_clone::<VecboostState>();
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_health_checks_return_healthy() {
use trait_kit::prelude::HealthStatus;
let state = make_app_state().await;
let embedding_health = state.kit.health_check::<EmbeddingModule>();
assert!(
embedding_health.is_ok(),
"EmbeddingModule health check should be registered"
);
assert_eq!(embedding_health.unwrap(), HealthStatus::Healthy);
let rerank_health = state.kit.health_check::<RerankModule>();
assert!(
rerank_health.is_ok(),
"RerankModule health check should be registered"
);
assert_eq!(rerank_health.unwrap(), HealthStatus::Healthy);
let rate_limit_health = state.kit.health_check::<RateLimitModule>();
assert!(
rate_limit_health.is_ok(),
"RateLimitModule health check should be registered"
);
assert_eq!(rate_limit_health.unwrap(), HealthStatus::Healthy);
let cache_health = state.kit.health_check::<CacheModule>();
assert!(
cache_health.is_ok(),
"CacheModule health check should be registered"
);
assert_eq!(
cache_health.unwrap(),
HealthStatus::Degraded {
detail: "cache disabled".into(),
}
);
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_kit_shutdown_completes_cleanly() {
let state = make_app_state().await;
state.kit.shutdown_async().await;
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_vecboost_state_new_and_kit_accessor() {
let state = make_app_state().await;
let kit_ref = state.kit();
assert_eq!(Arc::strong_count(kit_ref), 1);
assert!(kit_ref.contains::<EmbeddingModule>());
assert!(kit_ref.contains::<RerankModule>());
assert!(kit_ref.contains::<RateLimitModule>());
assert!(kit_ref.contains::<AuditModule>());
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_vecboost_state_constructor() {
let state = make_app_state().await;
let kit_clone = state.kit().clone();
let new_state = VecboostState::new(kit_clone);
assert!(new_state.kit.contains::<EmbeddingModule>());
}
#[cfg(feature = "http")]
#[tokio::test]
async fn test_mock_engine_direct_method_calls() {
let engine = MockEngine;
let vec = engine.embed("hello").unwrap();
assert_eq!(vec.len(), 384);
assert!(vec.iter().all(|&v| v == 0.0));
let texts = vec!["hello".to_string(), "world".to_string()];
let batch = engine.embed_batch(&texts).unwrap();
assert_eq!(batch.len(), 2);
assert_eq!(batch[0].len(), 384);
assert_eq!(*engine.precision(), Precision::Fp32);
assert!(!engine.supports_mixed_precision());
let config = crate::config::model::ModelConfig::default();
let mut engine_mut = MockEngine;
let result = engine_mut.try_fallback_to_cpu(&config).await;
assert!(result.is_ok());
}
#[cfg(all(feature = "http", feature = "auth"))]
#[tokio::test]
async fn test_from_ref_auth_config_returns_default() {
let state = make_app_state().await;
let auth_config: config::app::AuthConfig = FromRef::from_ref(&state);
let _ = auth_config;
}
}