oxirs_vec/
lib.rs

1//! # OxiRS Vector Search
2//!
3//! [![Version](https://img.shields.io/badge/version-0.4.1-blue)](https://github.com/cool-japan/oxirs/releases)
4//! [![docs.rs](https://docs.rs/oxirs-vec/badge.svg)](https://docs.rs/oxirs-vec)
5//!
6//! **Status**: Production Release (v0.4.1) - **Production-Ready with Complete Documentation**
7//! **Stability**: Public APIs are stable. Production-ready with comprehensive testing and 100 KB of documentation.
8//!
9//! Vector index abstractions for semantic similarity and AI-augmented SPARQL querying.
10//!
11//! This crate provides comprehensive vector search capabilities for knowledge graphs,
12//! enabling semantic similarity searches, AI-augmented SPARQL queries, and hybrid
13//! symbolic-vector operations.
14
15#![allow(dead_code)]
16//!
17//! ## Features
18//!
19//! - **Multi-algorithm embeddings**: TF-IDF, sentence transformers, custom models
20//! - **Advanced indexing**: HNSW, flat, quantized, and multi-index support
21//! - **Rich similarity metrics**: Cosine, Euclidean, Pearson, Jaccard, and more
22//! - **SPARQL integration**: `vec:similar` service functions and hybrid queries
23//! - **Performance optimization**: Caching, batching, and parallel processing
24//!
25//! ## Quick Start
26//!
27//! ```rust
28//! use oxirs_vec::{VectorStore, embeddings::EmbeddingStrategy};
29//!
30//! // Create vector store with sentence transformer embeddings
31//! let mut store = VectorStore::with_embedding_strategy(
32//!     EmbeddingStrategy::SentenceTransformer
33//! ).expect("should succeed");
34//!
35//! // Index some content
36//! store
37//!     .index_resource(
38//!         "http://example.org/doc1".to_string(),
39//!         "This is a document about AI",
40//!     )
41//!     .expect("should succeed");
42//! store
43//!     .index_resource(
44//!         "http://example.org/doc2".to_string(),
45//!         "Machine learning tutorial",
46//!     )
47//!     .expect("should succeed");
48//!
49//! // Search for similar content
50//! let results = store
51//!     .similarity_search("artificial intelligence", 5)
52//!     .expect("should succeed");
53//!
54//! println!("Found {} matching resources", results.len());
55//! ```
56//!
57//! ## Cargo Features
58//!
59//! This crate follows the **COOLJAPAN Pure Rust Policy**: default features are 100% Pure Rust
60//! with no C/Fortran/CUDA dependencies. Optional features requiring system libraries are
61//! properly feature-gated.
62//!
63//! ### Core Features (Pure Rust)
64//!
65//! - `simd` - SIMD optimizations for vector operations (Pure Rust)
66//! - `parallel` - Parallel processing support (Pure Rust)
67//! - `tantivy-search` - Tantivy full-text search (Pure Rust)
68//!
69//! HNSW index support is always built in (Pure Rust, no feature flag required).
70//!
71//! ### Optional Features (with system dependencies)
72//!
73//! - `gpu` - GPU acceleration abstractions (Pure Rust; no-op compatibility feature)
74//! - `blas` - BLAS acceleration (requires system BLAS library)
75//!
76//! > **CUDA**: Real NVIDIA CUDA GPU acceleration is provided by the companion
77//! > `oxirs-vec-adapter-cuda` crate (`publish = false`), which depends on
78//! > `oxirs-vec` and quarantines the `cuda-runtime-sys` C FFI off this crate's
79//! > Pure-Rust surface per the COOLJAPAN Pure Rust Policy v2. The former `cuda`
80//! > and `gpu-full` features were removed.
81//!
82//! ### Content Processing
83//!
84//! - `images` - Image processing support
85//! - `content-processing` - Full content processing (PDF, archives, XML, images)
86//!
87//! ### Language Integration
88//!
89//! - `python` - Python bindings via PyO3
90//!
91//! ### Default Build
92//!
93//! ```toml
94//! [dependencies]
95//! oxirs-vec = "0.1"  # 100% Pure Rust, no system dependencies
96//! ```
97//!
98//! ### GPU-Accelerated Build (requires CUDA toolkit)
99//!
100//! Real CUDA acceleration lives in the `oxirs-vec-adapter-cuda` crate
101//! (`publish = false`), which depends on `oxirs-vec` and is built only where the
102//! NVIDIA CUDA Toolkit is present.
103
104use anyhow::Result;
105
106pub mod adaptive_compression;
107pub mod adaptive_intelligent_caching;
108pub mod adaptive_recall_tuner;
109pub mod advanced_analytics;
110pub mod advanced_benchmarking;
111pub mod advanced_caching;
112pub mod advanced_caching_eviction;
113pub mod advanced_caching_multilevel;
114pub mod advanced_caching_worker;
115pub mod advanced_metrics;
116pub mod advanced_result_merging;
117pub mod automl_optimization;
118pub mod bench_metrics;
119pub mod bench_runner;
120pub mod bench_tests;
121pub mod benchmarking;
122pub mod cache_friendly_index;
123pub mod clustering;
124pub mod compaction;
125pub mod compression;
126pub mod compression_codecs;
127pub mod compression_io;
128#[cfg(test)]
129pub mod compression_tests;
130pub mod compression_types;
131#[cfg(feature = "content-processing")]
132pub mod content_processing;
133pub mod crash_recovery;
134pub mod cross_language_alignment;
135pub mod cross_modal_embeddings;
136pub mod delta_sync_store;
137pub mod diskann;
138pub mod distance_metrics;
139pub mod distributed;
140pub mod distributed_vector_search;
141pub mod dynamic_index_selector;
142pub mod embedding_pipeline;
143pub mod embeddings;
144pub mod enhanced_performance_monitoring;
145pub mod faiss_compatibility;
146pub mod faiss_gpu_integration;
147pub mod faiss_integration;
148pub mod faiss_migration_tools;
149pub mod faiss_native_integration;
150pub mod fault;
151pub mod federated_search;
152pub mod filtered_search;
153pub mod gnn_embeddings;
154pub mod gpu;
155pub mod gpu_benchmarks;
156pub mod gpu_hnsw_index;
157pub mod gpu_search_enhanced;
158pub mod graph_aware_search;
159pub mod graph_indices;
160pub mod hierarchical_similarity;
161pub mod hnsw;
162pub mod hnsw_persistence;
163pub mod huggingface;
164pub mod hybrid_fusion;
165pub mod hybrid_search;
166pub mod index;
167pub mod ivf;
168pub mod joint_embedding_spaces;
169pub mod joint_embedding_spaces_align;
170pub mod joint_embedding_spaces_aligner;
171pub mod joint_embedding_spaces_eval;
172#[cfg(test)]
173pub mod joint_embedding_spaces_tests;
174pub mod joint_embedding_spaces_transfer;
175pub mod joint_embedding_spaces_types;
176pub mod kg_embeddings;
177pub mod learned_index;
178pub mod lsh;
179pub mod mmap_advanced;
180pub mod mmap_index;
181pub mod multi_modal_search;
182pub mod multi_tenancy;
183pub mod nsg;
184pub mod opq;
185pub mod oxirs_arq_integration;
186pub mod performance_insights;
187pub mod persistence;
188pub mod personalized_search;
189pub mod pq;
190pub mod pq_index;
191pub mod pytorch;
192pub mod quantized_cache;
193pub mod quantum_search;
194pub mod query_planning;
195pub mod query_rewriter;
196pub mod random_utils;
197pub mod rdf_content_enhancement;
198pub mod rdf_integration;
199pub mod real_time_analytics;
200pub mod real_time_embedding_pipeline;
201pub mod real_time_updates;
202pub mod reranking;
203pub mod result_fusion;
204pub mod rta_aggregators;
205pub mod rta_engine;
206pub mod rta_tests;
207#[cfg(test)]
208mod score_contract_tests;
209pub mod similarity;
210pub mod sparql_integration;
211pub mod sparql_service_endpoint;
212pub mod sparse;
213pub mod sq;
214pub mod storage_optimizations;
215pub mod store_integration;
216pub(crate) mod store_integration_adapters;
217pub(crate) mod store_integration_sync;
218#[cfg(test)]
219mod store_integration_tests;
220pub mod store_integration_types;
221pub mod structured_vectors;
222pub mod tensorflow;
223pub mod tiering;
224pub mod tree_indices;
225pub mod tree_indices_balltree;
226pub mod tree_indices_covertree;
227pub mod tree_indices_kdtree;
228pub mod tree_indices_rptree;
229#[cfg(test)]
230mod tree_indices_tests;
231pub mod tree_indices_types;
232pub mod tree_indices_unified;
233pub mod tree_indices_vptree;
234pub mod validation;
235pub mod wal;
236pub mod word2vec;
237// Flat IVF approximate nearest-neighbour index (v1.1.0 round 5)
238pub mod flat_ivf_index;
239
240// LSH approximate nearest-neighbour index (v1.1.0 round 6)
241pub mod lsh_index;
242
243// IVF-PQ compound approximate nearest-neighbour index (v1.1.0 round 7)
244pub mod ivfpq_index;
245
246// HNSW ANN graph construction (v1.1.0 round 8)
247pub mod hnsw_builder;
248
249// Multi-vector product search combining multiple embedding sub-vectors (v1.1.0 round 9)
250pub mod product_search;
251
252// Vector quantization for embedding compression (v1.1.0 round 10)
253pub mod quantizer;
254
255// Delta encoding for incremental vector updates (v1.1.0 round 11)
256pub mod delta_encoder;
257
258// Vector embedding similarity metrics and nearest-neighbour utilities (v1.1.0 round 12)
259pub mod embedding_similarity;
260
261// HNSW approximate nearest-neighbor search (v1.1.0 round 13)
262pub mod hnsw_search;
263
264// Vector embedding cache with LRU eviction (v1.1.0 round 12)
265pub mod vector_cache;
266
267// ANN recall/latency benchmarking (v1.1.0 round 11)
268pub mod ann_benchmark;
269
270/// K-means clustering index: Lloyd's algorithm, cluster assignment, centroid tracking,
271/// cluster statistics, merge, split, ANN search by cluster probing (v1.1.0 round 13)
272pub mod cluster_index;
273
274/// ANN vector index merging: flat-index merge with last-write-wins dedup,
275/// filter, split, and merge statistics (v1.1.0 round 14)
276pub mod index_merger;
277
278/// Approximate cardinality counting using HyperLogLog (v1.1.0 round 15)
279pub mod approximate_counter;
280
281/// Product quantization encoder/decoder: PqConfig, PqEncoder with encode/decode/
282/// asymmetric_distance and random codebook initialisation (v1.1.0 round 16)
283pub mod pq_encoder;
284
285// Python bindings module
286#[cfg(feature = "python")]
287pub mod python_bindings;
288
289/// In-memory vector index and `VectorIndex` trait
290pub mod vector_index;
291
292/// Enhanced vector store with embedding management and persistence
293pub mod vector_store;
294
295/// Cost-based vector index optimizer (selectivity-aware family selection,
296/// online learning, persistent stats).  See [`optimizer`] for details.
297pub mod optimizer;
298
299/// Runtime index dispatcher: wraps the optimizer brain with concrete
300/// HNSW / IVF / LSH / PQ instances and re-issues queries on fallback.
301pub mod index_dispatcher;
302
303// Re-export types moved to dedicated modules
304pub use vector_index::{MemoryVectorIndex, VectorIndex};
305pub use vector_store::{
306    DocumentBatchProcessor, SearchOptions, SearchQuery, SearchType, VectorOperationResult,
307    VectorStore, VectorStoreConfig,
308};
309
310// Re-export commonly used types
311pub use adaptive_compression::{
312    AdaptiveCompressor, CompressionMetrics, CompressionPriorities, MultiLevelCompression,
313    VectorStats,
314};
315pub use adaptive_intelligent_caching::{
316    AccessPatternAnalyzer, AdaptiveIntelligentCache, CacheConfiguration, CacheOptimizer,
317    CachePerformanceMetrics, CacheTier, MLModels, PredictivePrefetcher,
318};
319pub use advanced_analytics::{
320    AnomalyDetection, AnomalyDetector, AnomalyType, ImplementationEffort,
321    OptimizationRecommendation, PerformanceTrends, Priority, QualityAspect, QualityRecommendation,
322    QueryAnalytics, QueryAnomaly, RecommendationType, VectorAnalyticsEngine,
323    VectorDistributionAnalysis, VectorQualityAssessment,
324};
325pub use advanced_benchmarking::{
326    AdvancedBenchmarkConfig, AdvancedBenchmarkResult, AdvancedBenchmarkSuite, AlgorithmParameters,
327    BenchmarkAlgorithm, BuildTimeMetrics, CacheMetrics, DatasetQualityMetrics, DatasetStatistics,
328    DistanceStatistics, EnhancedBenchmarkDataset, HyperparameterTuner, IndexSizeMetrics,
329    LatencyMetrics, MemoryMetrics, ObjectiveFunction, OptimizationStrategy,
330    ParallelBenchmarkConfig, ParameterSpace, ParameterType, ParameterValue, PerformanceMetrics,
331    PerformanceProfiler, QualityDegradation, QualityMetrics, ScalabilityMetrics,
332    StatisticalAnalyzer, StatisticalMetrics, ThroughputMetrics,
333};
334pub use advanced_caching::{
335    BackgroundCacheWorker, CacheAnalysisReport, CacheAnalyzer, CacheConfig, CacheEntry,
336    CacheInvalidator, CacheKey, CacheStats, CacheWarmer, EvictionPolicy, InvalidationStats,
337    MultiLevelCache, MultiLevelCacheStats,
338};
339pub use advanced_result_merging::{
340    AdvancedResultMerger, ConfidenceInterval, DiversityConfig, DiversityMetric, FusionStatistics,
341    MergedResult, RankFusionAlgorithm, RankingFactor, ResultExplanation, ResultMergingConfig,
342    ResultMetadata, ScoreCombinationStrategy, ScoreNormalizationMethod, ScoredResult,
343    SourceContribution, SourceResult, SourceType,
344};
345pub use automl_optimization::{
346    AutoMLConfig, AutoMLOptimizer, AutoMLResults, AutoMLStatistics, IndexConfiguration,
347    IndexParameterSpace, OptimizationMetric, OptimizationTrial, ResourceConstraints, SearchSpace,
348    TrialResult,
349};
350pub use benchmarking::{
351    BenchmarkConfig, BenchmarkDataset, BenchmarkOutputFormat, BenchmarkResult, BenchmarkRunner,
352    BenchmarkSuite, BenchmarkTestCase, MemoryMetrics as BenchmarkMemoryMetrics,
353    PerformanceMetrics as BenchmarkPerformanceMetrics, QualityMetrics as BenchmarkQualityMetrics,
354    ScalabilityMetrics as BenchmarkScalabilityMetrics, SystemInfo,
355};
356pub use cache_friendly_index::{CacheFriendlyVectorIndex, IndexConfig as CacheFriendlyIndexConfig};
357pub use compaction::{
358    CompactionConfig, CompactionManager, CompactionMetrics, CompactionResult, CompactionState,
359    CompactionStatistics, CompactionStrategy,
360};
361pub use compression::{create_compressor, CompressionMethod, VectorCompressor};
362#[cfg(feature = "content-processing")]
363pub use content_processing::{
364    ChunkType, ChunkingStrategy, ContentChunk, ContentExtractionConfig, ContentLocation,
365    ContentProcessor, DocumentFormat, DocumentStructure, ExtractedContent, ExtractedImage,
366    ExtractedLink, ExtractedTable, FormatHandler, Heading, ProcessingStats, TocEntry,
367};
368pub use crash_recovery::{CrashRecoveryManager, RecoveryConfig, RecoveryPolicy, RecoveryStats};
369pub use cross_modal_embeddings::{
370    AttentionMechanism, AudioData, AudioEncoder, CrossModalConfig, CrossModalEncoder, FusionLayer,
371    FusionStrategy, GraphData, GraphEncoder, ImageData, ImageEncoder, Modality, ModalityData,
372    MultiModalContent, TextEncoder, VideoData, VideoEncoder,
373};
374pub use diskann::{
375    DiskAnnBuildStats, DiskAnnBuilder, DiskAnnConfig, DiskAnnError, DiskAnnIndex, DiskAnnResult,
376    DiskStorage, IndexMetadata as DiskAnnIndexMetadata, MemoryMappedStorage, NodeId,
377    PruningStrategy, SearchMode as DiskAnnSearchMode, SearchStats as DiskAnnSearchStats,
378    StorageBackend, VamanaGraph, VamanaNode, VectorId as DiskAnnVectorId,
379};
380pub use distributed::{
381    // Raft consensus
382    AppendEntriesRequest,
383    AppendEntriesResponse,
384    ClusterSimulator,
385    // Cross-DC replication
386    ConflictRecord,
387    ConflictResolutionStrategy,
388    CrossDcConfig,
389    CrossDcCoordinator,
390    CrossDcStats,
391    IndexCommand,
392    NodeId as RaftNodeId,
393    NodeRole,
394    PrimaryDcManager,
395    RaftConfig,
396    RaftIndexNode,
397    RaftStats,
398    ReplicaDcManager,
399    ReplicaStatus,
400    ReplicationEntry,
401    ReplicationHealth,
402    ReplicationOperation,
403    ReplicationSeq,
404    RequestVoteRequest,
405    RequestVoteResponse,
406    Term,
407    VectorEntry as RaftVectorEntry,
408};
409pub use distributed_vector_search::{
410    ConsistencyLevel, DistributedClusterStats, DistributedNodeConfig, DistributedQuery,
411    DistributedSearchResponse, DistributedVectorSearch, LoadBalancingAlgorithm, NodeHealthStatus,
412    PartitioningStrategy, QueryExecutionStrategy,
413};
414pub use dynamic_index_selector::{DynamicIndexSelector, IndexSelectorConfig};
415pub use embedding_pipeline::{
416    DimensionalityReduction, EmbeddingPipeline, NormalizationConfig, PostprocessingPipeline,
417    PreprocessingPipeline, TokenizerConfig, VectorNormalization,
418};
419pub use embeddings::{
420    EmbeddableContent, EmbeddingConfig, EmbeddingManager, EmbeddingStrategy, ModelDetails,
421    OpenAIConfig, OpenAIEmbeddingGenerator, SentenceTransformerGenerator, TransformerModelType,
422};
423pub use enhanced_performance_monitoring::{
424    Alert, AlertManager, AlertSeverity, AlertThresholds, AlertType, AnalyticsEngine,
425    AnalyticsReport, DashboardData, EnhancedPerformanceMonitor, ExportConfig, ExportDestination,
426    ExportFormat, LatencyDistribution, MonitoringConfig as EnhancedMonitoringConfig,
427    QualityMetrics as EnhancedQualityMetrics, QualityMetricsCollector, QualityStatistics,
428    QueryInfo, QueryMetricsCollector, QueryStatistics, QueryType, Recommendation,
429    RecommendationCategory, RecommendationPriority, SystemMetrics, SystemMetricsCollector,
430    SystemStatistics, TrendData, TrendDirection,
431};
432pub use faiss_compatibility::{
433    CompressionLevel, ConversionMetrics, ConversionResult, FaissCompatibility, FaissExportConfig,
434    FaissImportConfig, FaissIndexMetadata, FaissIndexType, FaissMetricType, FaissParameter,
435    SimpleVectorIndex,
436};
437pub use federated_search::{
438    AuthenticationConfig, FederatedSearchConfig, FederatedVectorSearch, FederationEndpoint,
439    PrivacyEngine, PrivacyMode, SchemaCompatibility, TrustManager,
440};
441pub use gnn_embeddings::{AggregatorType, GraphSAGE, GCN};
442pub use gpu::{
443    create_default_accelerator,
444    create_memory_optimized_accelerator,
445    create_performance_accelerator,
446    is_gpu_available,
447    GpuAccelerator,
448    // GPU HNSW index builder (v0.2.0)
449    GpuBatchDistanceComputer,
450    GpuBuffer,
451    GpuConfig,
452    GpuDevice,
453    // Multi-GPU load balancing (v0.2.0)
454    GpuDeviceMetrics,
455    GpuDistanceMetric,
456    GpuExecutionConfig,
457    GpuHnswIndexBuilder,
458    GpuIndexBuildStats,
459    GpuIndexBuilderConfig,
460    GpuTaskOutput,
461    GpuTaskResult,
462    HnswGraph,
463    HnswNode,
464    IncrementalGpuIndexBuilder,
465    LoadBalancingStrategy,
466    MultiGpuConfig,
467    MultiGpuConfigFactory,
468    MultiGpuManager,
469    MultiGpuStats,
470    MultiGpuTask,
471    TaskPriority,
472};
473pub use gpu_benchmarks::{
474    BenchmarkResult as GpuBenchmarkResult, GpuBenchmarkConfig, GpuBenchmarkSuite,
475};
476pub use gpu_search_enhanced::{BatchSearchEngine, SearchMetrics, SimdVectorSearch};
477pub use graph_indices::{
478    DelaunayGraph, GraphIndex, GraphIndexConfig, GraphType, NSWGraph, ONNGGraph, PANNGGraph,
479    RNGGraph,
480};
481pub use hierarchical_similarity::{
482    ConceptHierarchy, HierarchicalSimilarity, HierarchicalSimilarityConfig,
483    HierarchicalSimilarityResult, HierarchicalSimilarityStats, SimilarityContext,
484    SimilarityExplanation, SimilarityTaskType,
485};
486pub use hnsw::{HnswConfig, HnswIndex};
487pub use hybrid_fusion::{
488    FusedResult, HybridFusion, HybridFusionConfig, HybridFusionStatistics, HybridFusionStrategy,
489    NormalizationMethod,
490};
491pub use hybrid_search::{
492    Bm25Scorer, DocumentScore, HybridQuery, HybridResult, HybridSearchConfig, HybridSearchManager,
493    KeywordAlgorithm, KeywordMatch, KeywordSearcher, QueryExpander, RankFusion, RankFusionStrategy,
494    SearchMode, SearchWeights, TfidfScorer,
495};
496
497#[cfg(feature = "tantivy-search")]
498pub use hybrid_search::{
499    IndexStats, RdfDocument, TantivyConfig, TantivySearchResult, TantivySearcher,
500};
501pub use index::{AdvancedVectorIndex, DistanceMetric, IndexConfig, IndexType, SearchResult};
502pub use ivf::{IvfConfig, IvfIndex, IvfStats, QuantizationStrategy};
503pub use joint_embedding_spaces::{
504    ActivationFunction, AlignmentPair, CLIPAligner, ContrastiveOptimizer, CrossModalAttention,
505    CurriculumLearning, DataAugmentation, DifficultySchedule, DomainAdapter, DomainStatistics,
506    JointEmbeddingConfig, JointEmbeddingSpace, LearningRateSchedule, LinearProjector,
507    PacingFunction, ScheduleType, TemperatureScheduler, TrainingStatistics,
508};
509pub use kg_embeddings::{
510    ComplEx, KGEmbedding, KGEmbeddingConfig, KGEmbeddingModel as KGModel, KGEmbeddingModelType,
511    RotatE, TransE, Triple,
512};
513pub use lsh::{LshConfig, LshFamily, LshIndex, LshStats};
514pub use mmap_index::{MemoryMappedIndexStats, MemoryMappedVectorIndex};
515pub use multi_tenancy::{
516    AccessControl, AccessPolicy, AdmissionController, AdmissionError, BillingEngine,
517    BillingMetrics, BillingPeriod, IsolationLevel, IsolationStrategy, MultiTenancyError,
518    MultiTenancyResult, MultiTenantManager, NamespaceManager, Permission, PricingModel,
519    PrioritizedQuery, QuotaEnforcer, QuotaLimits, QuotaUsage, RateLimiter, ResourceQuota,
520    ResourceType, Role, SlaClass, SlaQueryDispatcher, SlaThresholds, Tenant, TenantConfig,
521    TenantContext, TenantId, TenantManagerConfig, TenantMetadata, TenantOperation,
522    TenantStatistics, TenantStatus, UsageRecord,
523};
524pub use nsg::{DistanceMetric as NsgDistanceMetric, NsgConfig, NsgIndex, NsgStats};
525pub use performance_insights::{
526    AlertingSystem, OptimizationRecommendations, PerformanceInsightsAnalyzer,
527    PerformanceTrends as InsightsPerformanceTrends, QueryComplexity,
528    QueryStatistics as InsightsQueryStatistics, ReportFormat, VectorStatistics,
529};
530pub use persistence::{
531    apply_wal_entry, restore_to_timestamp, CheckpointRef, PointInTimeRestore, RestoreReport,
532};
533pub use pq::{PQConfig, PQIndex, PQStats};
534pub use pytorch::{
535    ArchitectureType, CompileMode, DeviceManager, PyTorchConfig, PyTorchDevice, PyTorchEmbedder,
536    PyTorchModelManager, PyTorchModelMetadata, PyTorchTokenizer,
537};
538pub use quantum_search::{
539    QuantumSearchConfig, QuantumSearchResult, QuantumSearchStatistics, QuantumState,
540    QuantumVectorSearch,
541};
542pub use query_planning::{
543    CostModel, IndexStatistics, QueryCharacteristics, QueryPlan, QueryPlanner, QueryStrategy,
544    VectorQueryType,
545};
546pub use query_rewriter::{
547    QueryRewriter, QueryRewriterConfig, QueryVectorStatistics, RewriteRule, RewrittenQuery,
548};
549pub use rdf_content_enhancement::{
550    ComponentWeights, MultiLanguageProcessor, PathConstraint, PathDirection, PropertyAggregator,
551    PropertyPath, RdfContentConfig, RdfContentProcessor, RdfContext, RdfEntity, RdfValue,
552    TemporalInfo,
553};
554pub use rdf_integration::{
555    RdfIntegrationStats, RdfTermMapping, RdfTermMetadata, RdfTermType, RdfVectorConfig,
556    RdfVectorIntegration, RdfVectorSearchResult, SearchMetadata,
557};
558pub use real_time_analytics::{
559    AlertSeverity as AnalyticsAlertSeverity, AlertType as AnalyticsAlertType, AnalyticsConfig,
560    AnalyticsEvent, AnalyticsReport as RealTimeAnalyticsReport,
561    DashboardData as RealTimeDashboardData, ExportFormat as AnalyticsExportFormat,
562    MetricsCollector, PerformanceMonitor, QueryMetrics, SystemMetrics as AnalyticsSystemMetrics,
563    VectorAnalyticsEngine as RealTimeVectorAnalyticsEngine,
564};
565pub use real_time_embedding_pipeline::{
566    AlertThresholds as PipelineAlertThresholds, AutoScalingConfig, CompressionConfig, ContentItem,
567    MonitoringConfig as PipelineMonitoringConfig, PipelineConfig as RealTimeEmbeddingConfig,
568    PipelineStatistics as PipelineStats, ProcessingPriority, ProcessingResult, ProcessingStatus,
569    RealTimeEmbeddingPipeline, VersioningStrategy,
570};
571pub use real_time_updates::{
572    BatchProcessor, RealTimeConfig, RealTimeVectorSearch, RealTimeVectorUpdater, UpdateBatch,
573    UpdateOperation, UpdatePriority, UpdateStats,
574};
575pub use reranking::{
576    CrossEncoder, CrossEncoderBackend, CrossEncoderModel, CrossEncoderReranker, DiversityReranker,
577    DiversityStrategy, FusionStrategy as RerankingFusionStrategy, ModelBackend, ModelConfig,
578    RerankingCache, RerankingCacheConfig, RerankingConfig, RerankingError, RerankingMode,
579    RerankingOutput, RerankingStats, Result as RerankingResult, ScoreFusion, ScoreFusionConfig,
580    ScoredCandidate,
581};
582pub use result_fusion::{
583    FusedResults, FusionAlgorithm, FusionConfig, FusionQualityMetrics, FusionStats,
584    ResultFusionEngine, ScoreNormalizationStrategy, SourceResults, VectorSearchResult,
585};
586pub use similarity::{AdaptiveSimilarity, SemanticSimilarity, SimilarityConfig, SimilarityMetric};
587pub use sparql_integration::{
588    CrossLanguageProcessor, FederatedQueryResult, QueryExecutor, SparqlVectorFunctions,
589    SparqlVectorService, VectorOperation, VectorQuery, VectorQueryResult, VectorServiceArg,
590    VectorServiceConfig, VectorServiceResult,
591};
592
593#[cfg(feature = "tantivy-search")]
594pub use sparql_integration::{RdfLiteral, SearchStats, SparqlSearchResult, SparqlTextFunctions};
595pub use sparql_service_endpoint::{
596    AuthenticationInfo, AuthenticationType, CustomFunctionRegistry, FederatedOperation,
597    FederatedSearchResult, FederatedServiceEndpoint, FederatedVectorQuery, FunctionMetadata,
598    LoadBalancer, ParameterInfo, ParameterType as ServiceParameterType, PartialSearchResult,
599    QueryScope, ReturnType, ServiceCapability, ServiceEndpointManager, ServiceType,
600};
601pub use sparse::{COOMatrix, CSRMatrix, SparseVector};
602pub use sq::{QuantizationMode, QuantizationParams, SqConfig, SqIndex, SqStats};
603pub use storage_optimizations::{
604    CompressionType, MmapVectorFile, StorageConfig, StorageUtils, VectorBlock, VectorFileHeader,
605    VectorReader, VectorWriter,
606};
607pub use structured_vectors::{
608    ConfidenceScoredVector, HierarchicalVector, NamedDimensionVector, TemporalVector,
609    WeightedDimensionVector,
610};
611pub use tensorflow::{
612    OptimizationLevel, PreprocessingPipeline as TensorFlowPreprocessingPipeline, ServerConfig,
613    SessionConfig, TensorDataType, TensorFlowConfig, TensorFlowDevice, TensorFlowEmbedder,
614    TensorFlowModelInfo, TensorFlowModelServer, TensorSpec,
615};
616pub use tiering::{
617    IndexMetadata, StorageTier, TierMetrics, TierStatistics, TierTransitionReason, TieringConfig,
618    TieringManager, TieringPolicy,
619};
620pub use tree_indices::{
621    BallTree, CoverTree, KdTree, RandomProjectionTree, TreeIndex, TreeIndexConfig, TreeType, VpTree,
622};
623pub use wal::{WalConfig, WalEntry, WalManager};
624pub use word2vec::{
625    AggregationMethod, OovStrategy, Word2VecConfig, Word2VecEmbeddingGenerator, Word2VecFormat,
626};
627
628// ---- Optimizer & runtime dispatcher (W2-S7) -------------------------------
629pub use index_dispatcher::{DispatchedSearch, IndexDispatcher, IndexDispatcherConfig};
630pub use optimizer::{
631    CostEstimate, CostModel as OptimizerCostModel, CostWeights, DispatchError, DispatchPlan,
632    DispatcherConfig as OptimizerDispatcherConfig, FamilyStats, IndexFamily, IndexParameters,
633    OptimizerDispatcher, QueryObservation, QueryStats, WorkloadProfile,
634};
635
636/// Vector identifier type
637pub type VectorId = String;
638
639/// Batch search result type
640pub type BatchSearchResult = Vec<Result<Vec<(String, f32)>>>;
641
642/// Trait for vector store implementations
643pub trait VectorStoreTrait: Send + Sync {
644    /// Insert a vector with metadata
645    fn insert_vector(&mut self, id: VectorId, vector: Vector) -> Result<()>;
646
647    /// Add a vector and return its ID
648    fn add_vector(&mut self, vector: Vector) -> Result<VectorId>;
649
650    /// Get a vector by its ID
651    fn get_vector(&self, id: &VectorId) -> Result<Option<Vector>>;
652
653    /// Get all vector IDs
654    fn get_all_vector_ids(&self) -> Result<Vec<VectorId>>;
655
656    /// Search for similar vectors
657    fn search_similar(&self, query: &Vector, k: usize) -> Result<Vec<(VectorId, f32)>>;
658
659    /// Remove a vector by ID
660    fn remove_vector(&mut self, id: &VectorId) -> Result<bool>;
661
662    /// Get the number of vectors stored
663    fn len(&self) -> usize;
664
665    /// Check if the store is empty
666    fn is_empty(&self) -> bool {
667        self.len() == 0
668    }
669}
670
671/// Precision types for vectors
672#[derive(Debug, Clone, Copy, PartialEq, serde::Serialize, serde::Deserialize)]
673pub enum VectorPrecision {
674    F32,
675    F64,
676    F16,
677    I8,
678    Binary,
679}
680
681/// Multi-precision vector with enhanced functionality
682#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
683pub struct Vector {
684    pub dimensions: usize,
685    pub precision: VectorPrecision,
686    pub values: VectorData,
687    pub metadata: Option<std::collections::HashMap<String, String>>,
688}
689
690/// Vector data storage supporting multiple precisions
691#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
692pub enum VectorData {
693    F32(Vec<f32>),
694    F64(Vec<f64>),
695    F16(Vec<u16>), // Using u16 to represent f16 bits
696    I8(Vec<i8>),
697    Binary(Vec<u8>), // Packed binary representation
698}
699
700impl Vector {
701    /// Create a new F32 vector from values
702    pub fn new(values: Vec<f32>) -> Self {
703        let dimensions = values.len();
704        Self {
705            dimensions,
706            precision: VectorPrecision::F32,
707            values: VectorData::F32(values),
708            metadata: None,
709        }
710    }
711
712    /// Create a new vector with specific precision
713    pub fn with_precision(values: VectorData) -> Self {
714        let (dimensions, precision) = match &values {
715            VectorData::F32(v) => (v.len(), VectorPrecision::F32),
716            VectorData::F64(v) => (v.len(), VectorPrecision::F64),
717            VectorData::F16(v) => (v.len(), VectorPrecision::F16),
718            VectorData::I8(v) => (v.len(), VectorPrecision::I8),
719            VectorData::Binary(v) => (v.len() * 8, VectorPrecision::Binary), // 8 bits per byte
720        };
721
722        Self {
723            dimensions,
724            precision,
725            values,
726            metadata: None,
727        }
728    }
729
730    /// Create a new vector with metadata
731    pub fn with_metadata(
732        values: Vec<f32>,
733        metadata: std::collections::HashMap<String, String>,
734    ) -> Self {
735        let dimensions = values.len();
736        Self {
737            dimensions,
738            precision: VectorPrecision::F32,
739            values: VectorData::F32(values),
740            metadata: Some(metadata),
741        }
742    }
743
744    /// Create F64 vector
745    pub fn f64(values: Vec<f64>) -> Self {
746        Self::with_precision(VectorData::F64(values))
747    }
748
749    /// Create F16 vector (using u16 representation)
750    pub fn f16(values: Vec<u16>) -> Self {
751        Self::with_precision(VectorData::F16(values))
752    }
753
754    /// Create I8 quantized vector
755    pub fn i8(values: Vec<i8>) -> Self {
756        Self::with_precision(VectorData::I8(values))
757    }
758
759    /// Create binary vector
760    pub fn binary(values: Vec<u8>) -> Self {
761        Self::with_precision(VectorData::Binary(values))
762    }
763
764    /// Get vector values as f32 (converting if necessary)
765    pub fn as_f32(&self) -> Vec<f32> {
766        match &self.values {
767            VectorData::F32(v) => v.clone(),
768            VectorData::F64(v) => v.iter().map(|&x| x as f32).collect(),
769            VectorData::F16(v) => v.iter().map(|&x| Self::f16_to_f32(x)).collect(),
770            VectorData::I8(v) => v.iter().map(|&x| x as f32 / 128.0).collect(), // Normalize to [-1, 1]
771            VectorData::Binary(v) => {
772                let mut result = Vec::new();
773                for &byte in v {
774                    for bit in 0..8 {
775                        result.push(if (byte >> bit) & 1 == 1 { 1.0 } else { 0.0 });
776                    }
777                }
778                result
779            }
780        }
781    }
782
783    /// Convert f32 to f16 representation (simplified)
784    #[allow(dead_code)]
785    fn f32_to_f16(value: f32) -> u16 {
786        // Simplified f16 conversion - in practice, use proper IEEE 754 half-precision
787        let bits = value.to_bits();
788        let sign = (bits >> 31) & 0x1;
789        let exp = ((bits >> 23) & 0xff) as i32;
790        let mantissa = bits & 0x7fffff;
791
792        // Simplified conversion
793        let f16_exp = if exp == 0 {
794            0
795        } else {
796            (exp - 127 + 15).clamp(0, 31) as u16
797        };
798
799        let f16_mantissa = (mantissa >> 13) as u16;
800        ((sign as u16) << 15) | (f16_exp << 10) | f16_mantissa
801    }
802
803    /// Convert f16 representation to f32 (simplified)
804    fn f16_to_f32(value: u16) -> f32 {
805        // Simplified f16 conversion - in practice, use proper IEEE 754 half-precision
806        let sign = (value >> 15) & 0x1;
807        let exp = ((value >> 10) & 0x1f) as i32;
808        let mantissa = value & 0x3ff;
809
810        if exp == 0 {
811            if mantissa == 0 {
812                if sign == 1 {
813                    -0.0
814                } else {
815                    0.0
816                }
817            } else {
818                // Denormalized number
819                let f32_exp = -14 - 127;
820                let f32_mantissa = (mantissa as u32) << 13;
821                f32::from_bits(((sign as u32) << 31) | ((f32_exp as u32) << 23) | f32_mantissa)
822            }
823        } else {
824            let f32_exp = exp - 15 + 127;
825            let f32_mantissa = (mantissa as u32) << 13;
826            f32::from_bits(((sign as u32) << 31) | ((f32_exp as u32) << 23) | f32_mantissa)
827        }
828    }
829
830    /// Quantize f32 vector to i8
831    pub fn quantize_to_i8(values: &[f32]) -> Vec<i8> {
832        // Find min/max for normalization
833        let min_val = values.iter().fold(f32::INFINITY, |a, &b| a.min(b));
834        let max_val = values.iter().fold(f32::NEG_INFINITY, |a, &b| a.max(b));
835        let range = max_val - min_val;
836
837        if range == 0.0 {
838            vec![0; values.len()]
839        } else {
840            values
841                .iter()
842                .map(|&x| {
843                    let normalized = (x - min_val) / range; // 0 to 1
844                    let scaled = normalized * 254.0 - 127.0; // -127 to 127
845                    scaled.round().clamp(-127.0, 127.0) as i8
846                })
847                .collect()
848        }
849    }
850
851    /// Convert to binary representation using threshold
852    pub fn to_binary(values: &[f32], threshold: f32) -> Vec<u8> {
853        let mut binary = Vec::new();
854        let mut current_byte = 0u8;
855        let mut bit_position = 0;
856
857        for &value in values {
858            if value > threshold {
859                current_byte |= 1 << bit_position;
860            }
861
862            bit_position += 1;
863            if bit_position == 8 {
864                binary.push(current_byte);
865                current_byte = 0;
866                bit_position = 0;
867            }
868        }
869
870        // Handle remaining bits
871        if bit_position > 0 {
872            binary.push(current_byte);
873        }
874
875        binary
876    }
877
878    /// Calculate cosine similarity with another vector
879    pub fn cosine_similarity(&self, other: &Vector) -> Result<f32> {
880        if self.dimensions != other.dimensions {
881            return Err(anyhow::anyhow!("Vector dimensions must match"));
882        }
883
884        let self_f32 = self.as_f32();
885        let other_f32 = other.as_f32();
886
887        let dot_product: f32 = self_f32.iter().zip(&other_f32).map(|(a, b)| a * b).sum();
888
889        let magnitude_self: f32 = self_f32.iter().map(|x| x * x).sum::<f32>().sqrt();
890        let magnitude_other: f32 = other_f32.iter().map(|x| x * x).sum::<f32>().sqrt();
891
892        if magnitude_self == 0.0 || magnitude_other == 0.0 {
893            return Ok(0.0);
894        }
895
896        Ok(dot_product / (magnitude_self * magnitude_other))
897    }
898
899    /// Calculate Euclidean distance to another vector
900    pub fn euclidean_distance(&self, other: &Vector) -> Result<f32> {
901        if self.dimensions != other.dimensions {
902            return Err(anyhow::anyhow!("Vector dimensions must match"));
903        }
904
905        let self_f32 = self.as_f32();
906        let other_f32 = other.as_f32();
907
908        let distance = self_f32
909            .iter()
910            .zip(&other_f32)
911            .map(|(a, b)| (a - b).powi(2))
912            .sum::<f32>()
913            .sqrt();
914
915        Ok(distance)
916    }
917
918    /// Calculate Manhattan distance (L1 norm) to another vector
919    pub fn manhattan_distance(&self, other: &Vector) -> Result<f32> {
920        if self.dimensions != other.dimensions {
921            return Err(anyhow::anyhow!("Vector dimensions must match"));
922        }
923
924        let self_f32 = self.as_f32();
925        let other_f32 = other.as_f32();
926
927        let distance = self_f32
928            .iter()
929            .zip(&other_f32)
930            .map(|(a, b)| (a - b).abs())
931            .sum();
932
933        Ok(distance)
934    }
935
936    /// Calculate Minkowski distance (general Lp norm) to another vector
937    pub fn minkowski_distance(&self, other: &Vector, p: f32) -> Result<f32> {
938        if self.dimensions != other.dimensions {
939            return Err(anyhow::anyhow!("Vector dimensions must match"));
940        }
941
942        if p <= 0.0 {
943            return Err(anyhow::anyhow!("p must be positive"));
944        }
945
946        let self_f32 = self.as_f32();
947        let other_f32 = other.as_f32();
948
949        if p == f32::INFINITY {
950            // Special case: Chebyshev distance
951            return self.chebyshev_distance(other);
952        }
953
954        let distance = self_f32
955            .iter()
956            .zip(&other_f32)
957            .map(|(a, b)| (a - b).abs().powf(p))
958            .sum::<f32>()
959            .powf(1.0 / p);
960
961        Ok(distance)
962    }
963
964    /// Calculate Chebyshev distance (L∞ norm) to another vector
965    pub fn chebyshev_distance(&self, other: &Vector) -> Result<f32> {
966        if self.dimensions != other.dimensions {
967            return Err(anyhow::anyhow!("Vector dimensions must match"));
968        }
969
970        let self_f32 = self.as_f32();
971        let other_f32 = other.as_f32();
972
973        let distance = self_f32
974            .iter()
975            .zip(&other_f32)
976            .map(|(a, b)| (a - b).abs())
977            .fold(0.0f32, |max, val| max.max(val));
978
979        Ok(distance)
980    }
981
982    /// Get vector magnitude (L2 norm)
983    pub fn magnitude(&self) -> f32 {
984        let values = self.as_f32();
985        values.iter().map(|x| x * x).sum::<f32>().sqrt()
986    }
987
988    /// Normalize vector to unit length
989    pub fn normalize(&mut self) {
990        let mag = self.magnitude();
991        if mag > 0.0 {
992            match &mut self.values {
993                VectorData::F32(values) => {
994                    for value in values {
995                        *value /= mag;
996                    }
997                }
998                VectorData::F64(values) => {
999                    let mag_f64 = mag as f64;
1000                    for value in values {
1001                        *value /= mag_f64;
1002                    }
1003                }
1004                _ => {
1005                    // For other types, convert to f32, normalize, then convert back
1006                    let mut f32_values = self.as_f32();
1007                    for value in &mut f32_values {
1008                        *value /= mag;
1009                    }
1010                    self.values = VectorData::F32(f32_values);
1011                    self.precision = VectorPrecision::F32;
1012                }
1013            }
1014        }
1015    }
1016
1017    /// Get a normalized copy of this vector
1018    pub fn normalized(&self) -> Vector {
1019        let mut normalized = self.clone();
1020        normalized.normalize();
1021        normalized
1022    }
1023
1024    /// Add another vector (element-wise)
1025    pub fn add(&self, other: &Vector) -> Result<Vector> {
1026        if self.dimensions != other.dimensions {
1027            return Err(anyhow::anyhow!("Vector dimensions must match"));
1028        }
1029
1030        let self_f32 = self.as_f32();
1031        let other_f32 = other.as_f32();
1032
1033        let result_values: Vec<f32> = self_f32
1034            .iter()
1035            .zip(&other_f32)
1036            .map(|(a, b)| a + b)
1037            .collect();
1038
1039        Ok(Vector::new(result_values))
1040    }
1041
1042    /// Subtract another vector (element-wise)
1043    pub fn subtract(&self, other: &Vector) -> Result<Vector> {
1044        if self.dimensions != other.dimensions {
1045            return Err(anyhow::anyhow!("Vector dimensions must match"));
1046        }
1047
1048        let self_f32 = self.as_f32();
1049        let other_f32 = other.as_f32();
1050
1051        let result_values: Vec<f32> = self_f32
1052            .iter()
1053            .zip(&other_f32)
1054            .map(|(a, b)| a - b)
1055            .collect();
1056
1057        Ok(Vector::new(result_values))
1058    }
1059
1060    /// Scale vector by a scalar
1061    pub fn scale(&self, scalar: f32) -> Vector {
1062        let values = self.as_f32();
1063        let scaled_values: Vec<f32> = values.iter().map(|x| x * scalar).collect();
1064
1065        Vector::new(scaled_values)
1066    }
1067
1068    /// Get the number of dimensions in the vector
1069    pub fn len(&self) -> usize {
1070        self.dimensions
1071    }
1072
1073    /// Check if vector is empty (zero dimensions)
1074    pub fn is_empty(&self) -> bool {
1075        self.dimensions == 0
1076    }
1077
1078    /// Get vector as slice of f32 values
1079    pub fn as_slice(&self) -> Vec<f32> {
1080        self.as_f32()
1081    }
1082}
1083
1084/// Error types specific to vector operations
1085#[derive(Debug, thiserror::Error)]
1086pub enum VectorError {
1087    #[error("Dimension mismatch: expected {expected}, got {actual}")]
1088    DimensionMismatch { expected: usize, actual: usize },
1089
1090    #[error("Empty vector")]
1091    EmptyVector,
1092
1093    #[error("Index not built")]
1094    IndexNotBuilt,
1095
1096    #[error("Embedding generation failed: {message}")]
1097    EmbeddingError { message: String },
1098
1099    #[error("SPARQL service error: {message}")]
1100    SparqlServiceError { message: String },
1101
1102    #[error("Compression error: {0}")]
1103    CompressionError(String),
1104
1105    #[error("Invalid dimensions: {0}")]
1106    InvalidDimensions(String),
1107
1108    #[error("Unsupported operation: {0}")]
1109    UnsupportedOperation(String),
1110
1111    #[error("Invalid data: {0}")]
1112    InvalidData(String),
1113
1114    #[error("IO error: {0}")]
1115    IoError(#[from] std::io::Error),
1116}
1117
1118/// Utility functions for vector operations
1119pub mod utils {
1120    use super::Vector;
1121
1122    /// Calculate centroid of a set of vectors
1123    pub fn centroid(vectors: &[Vector]) -> Option<Vector> {
1124        if vectors.is_empty() {
1125            return None;
1126        }
1127
1128        let dimensions = vectors[0].dimensions;
1129        let mut sum_values = vec![0.0; dimensions];
1130
1131        for vector in vectors {
1132            if vector.dimensions != dimensions {
1133                return None; // Inconsistent dimensions
1134            }
1135
1136            let vector_f32 = vector.as_f32();
1137            for (i, &value) in vector_f32.iter().enumerate() {
1138                sum_values[i] += value;
1139            }
1140        }
1141
1142        let count = vectors.len() as f32;
1143        for value in &mut sum_values {
1144            *value /= count;
1145        }
1146
1147        Some(Vector::new(sum_values))
1148    }
1149
1150    /// Generate random vector for testing
1151    pub fn random_vector(dimensions: usize, seed: Option<u64>) -> Vector {
1152        use std::collections::hash_map::DefaultHasher;
1153        use std::hash::{Hash, Hasher};
1154
1155        let mut hasher = DefaultHasher::new();
1156        seed.unwrap_or(42).hash(&mut hasher);
1157        let mut rng_state = hasher.finish();
1158
1159        let mut values = Vec::with_capacity(dimensions);
1160        for _ in 0..dimensions {
1161            rng_state = rng_state.wrapping_mul(1103515245).wrapping_add(12345);
1162            let normalized = (rng_state as f32) / (u64::MAX as f32);
1163            values.push((normalized - 0.5) * 2.0); // Range: -1.0 to 1.0
1164        }
1165
1166        Vector::new(values)
1167    }
1168
1169    /// Convert vector to normalized unit vector
1170    pub fn normalize_vector(vector: &Vector) -> Vector {
1171        vector.normalized()
1172    }
1173}
1174
1175#[cfg(test)]
1176mod tests {
1177    use super::*;
1178    use crate::similarity::SimilarityMetric;
1179
1180    #[test]
1181    fn test_vector_creation() {
1182        let values = vec![1.0, 2.0, 3.0];
1183        let vector = Vector::new(values.clone());
1184
1185        assert_eq!(vector.dimensions, 3);
1186        assert_eq!(vector.precision, VectorPrecision::F32);
1187        assert_eq!(vector.as_f32(), values);
1188    }
1189
1190    #[test]
1191    fn test_multi_precision_vectors() {
1192        // Test F64 vector
1193        let f64_values = vec![1.0, 2.0, 3.0];
1194        let f64_vector = Vector::f64(f64_values.clone());
1195        assert_eq!(f64_vector.precision, VectorPrecision::F64);
1196        assert_eq!(f64_vector.dimensions, 3);
1197
1198        // Test I8 vector
1199        let i8_values = vec![100, -50, 0];
1200        let i8_vector = Vector::i8(i8_values);
1201        assert_eq!(i8_vector.precision, VectorPrecision::I8);
1202        assert_eq!(i8_vector.dimensions, 3);
1203
1204        // Test binary vector
1205        let binary_values = vec![0b10101010, 0b11110000];
1206        let binary_vector = Vector::binary(binary_values);
1207        assert_eq!(binary_vector.precision, VectorPrecision::Binary);
1208        assert_eq!(binary_vector.dimensions, 16); // 2 bytes * 8 bits
1209    }
1210
1211    #[test]
1212    fn test_vector_operations() -> Result<()> {
1213        let v1 = Vector::new(vec![1.0, 2.0, 3.0]);
1214        let v2 = Vector::new(vec![4.0, 5.0, 6.0]);
1215
1216        // Test addition
1217        let sum = v1.add(&v2)?;
1218        assert_eq!(sum.as_f32(), vec![5.0, 7.0, 9.0]);
1219
1220        // Test subtraction
1221        let diff = v2.subtract(&v1)?;
1222        assert_eq!(diff.as_f32(), vec![3.0, 3.0, 3.0]);
1223
1224        // Test scaling
1225        let scaled = v1.scale(2.0);
1226        assert_eq!(scaled.as_f32(), vec![2.0, 4.0, 6.0]);
1227        Ok(())
1228    }
1229
1230    #[test]
1231    fn test_cosine_similarity() -> Result<()> {
1232        let v1 = Vector::new(vec![1.0, 0.0, 0.0]);
1233        let v2 = Vector::new(vec![1.0, 0.0, 0.0]);
1234        let v3 = Vector::new(vec![0.0, 1.0, 0.0]);
1235
1236        // Identical vectors should have similarity 1.0
1237        assert!((v1.cosine_similarity(&v2).expect("test value") - 1.0).abs() < 0.001);
1238
1239        // Orthogonal vectors should have similarity 0.0
1240        assert!((v1.cosine_similarity(&v3).expect("test value")).abs() < 0.001);
1241        Ok(())
1242    }
1243
1244    #[test]
1245    fn test_vector_store() -> Result<()> {
1246        let mut store = VectorStore::new();
1247
1248        // Test indexing
1249        store.index_resource("doc1".to_string(), "This is a test")?;
1250        store.index_resource("doc2".to_string(), "Another test document")?;
1251
1252        // Test searching
1253        let results = store.similarity_search("test", 5)?;
1254        assert_eq!(results.len(), 2);
1255
1256        // Results should be sorted by similarity (descending)
1257        assert!(results[0].1 >= results[1].1);
1258        Ok(())
1259    }
1260
1261    #[test]
1262    fn test_similarity_metrics() -> Result<()> {
1263        let a = vec![1.0, 2.0, 3.0];
1264        let b = vec![4.0, 5.0, 6.0];
1265
1266        // Test different similarity metrics
1267        let cosine_sim = SimilarityMetric::Cosine.similarity(&a, &b)?;
1268        let euclidean_sim = SimilarityMetric::Euclidean.similarity(&a, &b)?;
1269        let manhattan_sim = SimilarityMetric::Manhattan.similarity(&a, &b)?;
1270
1271        // All similarities should be between 0 and 1
1272        assert!((0.0..=1.0).contains(&cosine_sim));
1273        assert!((0.0..=1.0).contains(&euclidean_sim));
1274        assert!((0.0..=1.0).contains(&manhattan_sim));
1275        Ok(())
1276    }
1277
1278    #[test]
1279    fn test_quantization() {
1280        let values = vec![1.0, -0.5, 0.0, 0.75];
1281        let quantized = Vector::quantize_to_i8(&values);
1282
1283        // Check that quantized values are in the expected range
1284        for &q in &quantized {
1285            assert!((-127..=127).contains(&q));
1286        }
1287    }
1288
1289    #[test]
1290    fn test_binary_conversion() {
1291        let values = vec![0.8, -0.3, 0.1, -0.9];
1292        let binary = Vector::to_binary(&values, 0.0);
1293
1294        // Should have 1 byte (4 values, each becomes 1 bit, packed into bytes)
1295        assert_eq!(binary.len(), 1);
1296
1297        // First bit should be 1 (0.8 > 0.0), second should be 0 (-0.3 < 0.0), etc.
1298        let byte = binary[0];
1299        assert_eq!(byte & 1, 1); // bit 0: 0.8 > 0.0
1300        assert_eq!((byte >> 1) & 1, 0); // bit 1: -0.3 < 0.0
1301        assert_eq!((byte >> 2) & 1, 1); // bit 2: 0.1 > 0.0
1302        assert_eq!((byte >> 3) & 1, 0); // bit 3: -0.9 < 0.0
1303    }
1304
1305    #[test]
1306    fn test_memory_vector_index() -> Result<()> {
1307        let mut index = MemoryVectorIndex::new();
1308
1309        let v1 = Vector::new(vec![1.0, 0.0, 0.0]);
1310        let v2 = Vector::new(vec![0.0, 1.0, 0.0]);
1311
1312        index.insert("v1".to_string(), v1.clone())?;
1313        index.insert("v2".to_string(), v2.clone())?;
1314
1315        // Test KNN search
1316        let results = index.search_knn(&v1, 1)?;
1317        assert_eq!(results.len(), 1);
1318        assert_eq!(results[0].0, "v1");
1319
1320        // Test threshold search
1321        let results = index.search_threshold(&v1, 0.5)?;
1322        assert!(!results.is_empty());
1323        Ok(())
1324    }
1325
1326    #[test]
1327    fn test_hnsw_index() -> Result<()> {
1328        use crate::hnsw::{HnswConfig, HnswIndex};
1329
1330        let config = HnswConfig::default();
1331        let mut index = HnswIndex::new(config)?;
1332
1333        let v1 = Vector::new(vec![1.0, 0.0, 0.0]);
1334        let v2 = Vector::new(vec![0.0, 1.0, 0.0]);
1335        let v3 = Vector::new(vec![0.0, 0.0, 1.0]);
1336
1337        index.insert("v1".to_string(), v1.clone())?;
1338        index.insert("v2".to_string(), v2.clone())?;
1339        index.insert("v3".to_string(), v3.clone())?;
1340
1341        // Test KNN search
1342        let results = index.search_knn(&v1, 2)?;
1343        assert!(results.len() <= 2);
1344
1345        // The first result should be v1 itself (highest similarity)
1346        if !results.is_empty() {
1347            assert_eq!(results[0].0, "v1");
1348        }
1349        Ok(())
1350    }
1351
1352    #[test]
1353    fn test_save_load_roundtrip() -> Result<()> {
1354        let dir = std::env::temp_dir().join(format!("oxirs_vec_test_{}", uuid::Uuid::new_v4()));
1355
1356        // Build a store with three known vectors.
1357        let mut store = VectorStore::new();
1358        let v1 = Vector::new(vec![1.0, 0.0, 0.0]);
1359        let v2 = Vector::new(vec![0.0, 1.0, 0.0]);
1360        let v3 = Vector::new(vec![0.0, 0.0, 1.0]);
1361
1362        store.index_vector("alpha".to_string(), v1.clone())?;
1363        store.index_vector("beta".to_string(), v2.clone())?;
1364        store.index_vector("gamma".to_string(), v3.clone())?;
1365
1366        // Save.
1367        let path = dir
1368            .to_str()
1369            .ok_or_else(|| anyhow::anyhow!("temp dir path is not UTF-8"))?;
1370        store.save_to_disk(path)?;
1371
1372        // Load into a fresh store.
1373        let loaded = VectorStore::load_from_disk(path)?;
1374
1375        // Verify each vector survives the roundtrip by exact retrieval.
1376        let r_alpha = loaded.get_vector("alpha").expect("alpha must be present");
1377        assert_eq!(r_alpha.as_f32(), v1.as_f32(), "alpha roundtrip mismatch");
1378
1379        let r_beta = loaded.get_vector("beta").expect("beta must be present");
1380        assert_eq!(r_beta.as_f32(), v2.as_f32(), "beta roundtrip mismatch");
1381
1382        let r_gamma = loaded.get_vector("gamma").expect("gamma must be present");
1383        assert_eq!(r_gamma.as_f32(), v3.as_f32(), "gamma roundtrip mismatch");
1384
1385        // Verify search still works: query aligned with v1 should rank "alpha" first.
1386        let results = loaded.similarity_search_vector(&v1, 3)?;
1387        assert!(!results.is_empty(), "search returned no results after load");
1388        assert_eq!(
1389            results[0].0, "alpha",
1390            "top result after load should be alpha"
1391        );
1392
1393        // Clean up.
1394        let _ = std::fs::remove_dir_all(&dir);
1395        Ok(())
1396    }
1397
1398    #[test]
1399    fn test_sparql_vector_service() -> Result<()> {
1400        use crate::embeddings::EmbeddingStrategy;
1401        use crate::sparql_integration::{
1402            SparqlVectorService, VectorServiceArg, VectorServiceConfig, VectorServiceResult,
1403        };
1404
1405        let config = VectorServiceConfig::default();
1406        let mut service = SparqlVectorService::new(config, EmbeddingStrategy::SentenceTransformer)?;
1407
1408        // Test vector similarity function
1409        let v1 = Vector::new(vec![1.0, 0.0, 0.0]);
1410        let v2 = Vector::new(vec![1.0, 0.0, 0.0]);
1411
1412        let args = vec![VectorServiceArg::Vector(v1), VectorServiceArg::Vector(v2)];
1413
1414        let result = service.execute_function("vector_similarity", &args)?;
1415
1416        match result {
1417            VectorServiceResult::Number(similarity) => {
1418                assert!((similarity - 1.0).abs() < 0.001); // Should be very similar
1419            }
1420            _ => panic!("Expected a number result"),
1421        }
1422
1423        // Test text embedding function
1424        let text_args = vec![VectorServiceArg::String("test text".to_string())];
1425        let embed_result = service.execute_function("embed_text", &text_args)?;
1426
1427        match embed_result {
1428            VectorServiceResult::Vector(vector) => {
1429                assert_eq!(vector.dimensions, 384); // Default embedding size
1430            }
1431            _ => panic!("Expected a vector result"),
1432        }
1433        Ok(())
1434    }
1435}
oxirs_vec/lib.rs

oxirs_vec/
lib.rs
