oxirs_vec/tiering/
types.rs

1//! Core types for hot/warm/cold tiering system
2
3use serde::{Deserialize, Serialize};
4use std::collections::HashMap;
5use std::path::PathBuf;
6use std::time::{Duration, SystemTime};
7
8/// Storage tier classification
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
10pub enum StorageTier {
11    /// In-memory storage (fastest, most expensive)
12    Hot,
13    /// Memory-mapped or SSD storage (moderate speed and cost)
14    Warm,
15    /// Compressed disk storage (slowest, cheapest)
16    Cold,
17}
18
19impl StorageTier {
20    /// Get the typical latency for this tier
21    pub fn typical_latency(&self) -> Duration {
22        match self {
23            StorageTier::Hot => Duration::from_micros(100), // Sub-millisecond
24            StorageTier::Warm => Duration::from_millis(5),  // Few milliseconds
25            StorageTier::Cold => Duration::from_millis(100), // ~100ms for decompression + load
26        }
27    }
28
29    /// Get the relative cost factor for this tier
30    pub fn cost_factor(&self) -> f64 {
31        match self {
32            StorageTier::Hot => 10.0, // Most expensive (RAM)
33            StorageTier::Warm => 2.0, // Moderate (SSD)
34            StorageTier::Cold => 1.0, // Cheapest (HDD/compressed)
35        }
36    }
37
38    /// Get the compression ratio for this tier
39    pub fn compression_ratio(&self) -> f64 {
40        match self {
41            StorageTier::Hot => 1.0,  // No compression
42            StorageTier::Warm => 1.2, // Light compression
43            StorageTier::Cold => 4.0, // Heavy compression
44        }
45    }
46}
47
48/// Metadata about an indexed dataset
49#[derive(Debug, Clone, Serialize, Deserialize)]
50pub struct IndexMetadata {
51    /// Unique identifier for this index
52    pub index_id: String,
53    /// Current storage tier
54    pub current_tier: StorageTier,
55    /// Size in bytes (uncompressed)
56    pub size_bytes: u64,
57    /// Size in bytes (compressed, if applicable)
58    pub compressed_size_bytes: u64,
59    /// Number of vectors in this index
60    pub vector_count: usize,
61    /// Vector dimensionality
62    pub dimension: usize,
63    /// Index type (HNSW, IVF, etc.)
64    pub index_type: IndexType,
65    /// Creation timestamp
66    pub created_at: SystemTime,
67    /// Last access timestamp
68    pub last_accessed: SystemTime,
69    /// Last modification timestamp
70    pub last_modified: SystemTime,
71    /// Access statistics
72    pub access_stats: AccessStatistics,
73    /// Performance metrics
74    pub performance_metrics: PerformanceMetrics,
75    /// Storage location
76    pub storage_path: Option<PathBuf>,
77    /// Custom metadata
78    pub custom_metadata: HashMap<String, String>,
79}
80
81/// Index type
82#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
83pub enum IndexType {
84    Hnsw,
85    Ivf,
86    Pq,
87    Sq,
88    Lsh,
89    Nsg,
90    Flat,
91}
92
93/// Access statistics for an index
94#[derive(Debug, Clone, Default, Serialize, Deserialize)]
95pub struct AccessStatistics {
96    /// Total number of queries to this index
97    pub total_queries: u64,
98    /// Number of queries in the last hour
99    pub queries_last_hour: u64,
100    /// Number of queries in the last day
101    pub queries_last_day: u64,
102    /// Number of queries in the last week
103    pub queries_last_week: u64,
104    /// Average queries per second (moving average)
105    pub avg_qps: f64,
106    /// Peak queries per second
107    pub peak_qps: f64,
108    /// Last access timestamp
109    pub last_access_time: Option<SystemTime>,
110    /// Access pattern (hot, warm, cold)
111    pub access_pattern: AccessPattern,
112    /// Query latencies (p50, p95, p99 in microseconds)
113    pub query_latencies: LatencyPercentiles,
114}
115
116/// Access pattern classification
117#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
118pub enum AccessPattern {
119    /// Frequently accessed, high QPS
120    Hot,
121    /// Moderately accessed
122    Warm,
123    /// Rarely accessed
124    Cold,
125    /// Bursty access pattern
126    Bursty,
127    /// Seasonal pattern
128    Seasonal,
129    /// Unknown pattern (insufficient data)
130    #[default]
131    Unknown,
132}
133
134/// Latency percentiles
135#[derive(Debug, Clone, Default, Serialize, Deserialize)]
136pub struct LatencyPercentiles {
137    pub p50: u64,
138    pub p95: u64,
139    pub p99: u64,
140    pub max: u64,
141}
142
143/// Performance metrics for an index
144#[derive(Debug, Clone, Default, Serialize, Deserialize)]
145pub struct PerformanceMetrics {
146    /// Average query latency in microseconds
147    pub avg_query_latency_us: u64,
148    /// Average load time in milliseconds
149    pub avg_load_time_ms: u64,
150    /// Cache hit rate (0.0 - 1.0)
151    pub cache_hit_rate: f64,
152    /// Memory footprint in bytes
153    pub memory_footprint_bytes: u64,
154    /// CPU utilization (0.0 - 1.0)
155    pub cpu_utilization: f64,
156    /// IO operations per second
157    pub iops: f64,
158    /// Throughput in MB/s
159    pub throughput_mbps: f64,
160}
161
162/// Statistics for a storage tier
163#[derive(Debug, Clone, Serialize, Deserialize)]
164pub struct TierStatistics {
165    /// Total capacity in bytes
166    pub capacity_bytes: u64,
167    /// Used capacity in bytes
168    pub used_bytes: u64,
169    /// Number of indices in this tier
170    pub index_count: usize,
171    /// Total number of queries to this tier
172    pub total_queries: u64,
173    /// Average query latency in microseconds
174    pub avg_query_latency_us: u64,
175    /// Hit rate for this tier
176    pub hit_rate: f64,
177    /// Number of promotions from this tier
178    pub promotions: u64,
179    /// Number of demotions to this tier
180    pub demotions: u64,
181    /// Total bytes read
182    pub bytes_read: u64,
183    /// Total bytes written
184    pub bytes_written: u64,
185    /// Last update timestamp
186    pub last_updated: SystemTime,
187}
188
189impl Default for TierStatistics {
190    fn default() -> Self {
191        Self {
192            capacity_bytes: 0,
193            used_bytes: 0,
194            index_count: 0,
195            total_queries: 0,
196            avg_query_latency_us: 0,
197            hit_rate: 0.0,
198            promotions: 0,
199            demotions: 0,
200            bytes_read: 0,
201            bytes_written: 0,
202            last_updated: SystemTime::now(),
203        }
204    }
205}
206
207impl TierStatistics {
208    /// Calculate utilization ratio (0.0 - 1.0)
209    pub fn utilization(&self) -> f64 {
210        if self.capacity_bytes == 0 {
211            0.0
212        } else {
213            self.used_bytes as f64 / self.capacity_bytes as f64
214        }
215    }
216
217    /// Calculate available bytes
218    pub fn available_bytes(&self) -> u64 {
219        self.capacity_bytes.saturating_sub(self.used_bytes)
220    }
221
222    /// Check if tier is near capacity
223    pub fn is_near_capacity(&self, threshold: f64) -> bool {
224        self.utilization() >= threshold
225    }
226}
227
228/// Tier transition event
229#[derive(Debug, Clone, Serialize, Deserialize)]
230pub struct TierTransition {
231    /// Index identifier
232    pub index_id: String,
233    /// Source tier
234    pub from_tier: StorageTier,
235    /// Destination tier
236    pub to_tier: StorageTier,
237    /// Reason for transition
238    pub reason: String,
239    /// Timestamp of transition
240    pub timestamp: SystemTime,
241    /// Transition duration
242    pub duration: Duration,
243    /// Success flag
244    pub success: bool,
245    /// Error message (if failed)
246    pub error: Option<String>,
247}
248
249/// Configuration for gradual tier transitions
250#[derive(Debug, Clone, Serialize, Deserialize)]
251pub struct GradualTransitionConfig {
252    /// Enable gradual transitions
253    pub enabled: bool,
254    /// Number of stages for gradual transition
255    pub stages: usize,
256    /// Delay between stages
257    pub stage_delay: Duration,
258    /// Monitor performance during transition
259    pub monitor_performance: bool,
260    /// Rollback on performance degradation
261    pub rollback_on_degradation: bool,
262    /// Performance degradation threshold (0.0 - 1.0)
263    pub degradation_threshold: f64,
264}
265
266impl Default for GradualTransitionConfig {
267    fn default() -> Self {
268        Self {
269            enabled: true,
270            stages: 3,
271            stage_delay: Duration::from_secs(60),
272            monitor_performance: true,
273            rollback_on_degradation: true,
274            degradation_threshold: 0.2, // 20% degradation triggers rollback
275        }
276    }
277}
278
279/// Cost model for tier placement
280#[derive(Debug, Clone, Serialize, Deserialize)]
281pub struct TierCostModel {
282    /// Cost per GB per hour for hot tier
283    pub hot_cost_per_gb_hour: f64,
284    /// Cost per GB per hour for warm tier
285    pub warm_cost_per_gb_hour: f64,
286    /// Cost per GB per hour for cold tier
287    pub cold_cost_per_gb_hour: f64,
288    /// Cost per query for hot tier
289    pub hot_query_cost: f64,
290    /// Cost per query for warm tier
291    pub warm_query_cost: f64,
292    /// Cost per query for cold tier
293    pub cold_query_cost: f64,
294    /// Cost per tier transition
295    pub transition_cost: f64,
296}
297
298impl Default for TierCostModel {
299    fn default() -> Self {
300        Self {
301            hot_cost_per_gb_hour: 0.10,   // High cost for RAM
302            warm_cost_per_gb_hour: 0.02,  // Moderate cost for SSD
303            cold_cost_per_gb_hour: 0.005, // Low cost for HDD
304            hot_query_cost: 0.0001,       // Low per-query cost (fast)
305            warm_query_cost: 0.0005,      // Higher per-query cost (slower)
306            cold_query_cost: 0.002,       // Highest per-query cost (slowest)
307            transition_cost: 0.01,        // Cost of moving data between tiers
308        }
309    }
310}
311
312/// Resource allocation for multi-tenancy
313#[derive(Debug, Clone, Serialize, Deserialize)]
314pub struct TenantResourceAllocation {
315    /// Tenant identifier
316    pub tenant_id: String,
317    /// Allocated capacity in hot tier (bytes)
318    pub hot_tier_allocation_bytes: u64,
319    /// Allocated capacity in warm tier (bytes)
320    pub warm_tier_allocation_bytes: u64,
321    /// Allocated capacity in cold tier (bytes)
322    pub cold_tier_allocation_bytes: u64,
323    /// QPS quota
324    pub qps_quota: f64,
325    /// Priority (higher values = higher priority)
326    pub priority: u32,
327    /// Custom policies
328    pub custom_policies: HashMap<String, String>,
329}
oxirs_vec/tiering/types.rs

oxirs_vec/tiering/
types.rs
