oxirs_vec/tiering/
types.rs

1//! Core types for hot/warm/cold tiering system
2
3use serde::{Deserialize, Serialize};
4use std::collections::HashMap;
5use std::path::PathBuf;
6use std::time::{Duration, SystemTime};
7
8/// Storage tier classification
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
10pub enum StorageTier {
11    /// In-memory storage (fastest, most expensive)
12    Hot,
13    /// Memory-mapped or SSD storage (moderate speed and cost)
14    Warm,
15    /// Compressed disk storage (slowest, cheapest)
16    Cold,
17}
18
19impl StorageTier {
20    /// Get the typical latency for this tier
21    pub fn typical_latency(&self) -> Duration {
22        match self {
23            StorageTier::Hot => Duration::from_micros(100), // Sub-millisecond
24            StorageTier::Warm => Duration::from_millis(5),  // Few milliseconds
25            StorageTier::Cold => Duration::from_millis(100), // ~100ms for decompression + load
26        }
27    }
28
29    /// Get the relative cost factor for this tier
30    pub fn cost_factor(&self) -> f64 {
31        match self {
32            StorageTier::Hot => 10.0, // Most expensive (RAM)
33            StorageTier::Warm => 2.0, // Moderate (SSD)
34            StorageTier::Cold => 1.0, // Cheapest (HDD/compressed)
35        }
36    }
37
38    /// Get the compression ratio for this tier
39    pub fn compression_ratio(&self) -> f64 {
40        match self {
41            StorageTier::Hot => 1.0,  // No compression
42            StorageTier::Warm => 1.2, // Light compression
43            StorageTier::Cold => 4.0, // Heavy compression
44        }
45    }
46}
47
48/// Metadata about an indexed dataset
49#[derive(Debug, Clone, Serialize, Deserialize)]
50pub struct IndexMetadata {
51    /// Unique identifier for this index
52    pub index_id: String,
53    /// Current storage tier
54    pub current_tier: StorageTier,
55    /// Size in bytes (uncompressed)
56    pub size_bytes: u64,
57    /// Size in bytes (compressed, if applicable)
58    pub compressed_size_bytes: u64,
59    /// Number of vectors in this index
60    pub vector_count: usize,
61    /// Vector dimensionality
62    pub dimension: usize,
63    /// Index type (HNSW, IVF, etc.)
64    pub index_type: IndexType,
65    /// Creation timestamp
66    pub created_at: SystemTime,
67    /// Last access timestamp
68    pub last_accessed: SystemTime,
69    /// Last modification timestamp
70    pub last_modified: SystemTime,
71    /// Access statistics
72    pub access_stats: AccessStatistics,
73    /// Performance metrics
74    pub performance_metrics: PerformanceMetrics,
75    /// Storage location
76    pub storage_path: Option<PathBuf>,
77    /// Custom metadata
78    pub custom_metadata: HashMap<String, String>,
79}
80
81/// Index type
82#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
83pub enum IndexType {
84    Hnsw,
85    Ivf,
86    Pq,
87    Sq,
88    Lsh,
89    Nsg,
90    Flat,
91}
92
93/// Access statistics for an index
94#[derive(Debug, Clone, Default, Serialize, Deserialize)]
95pub struct AccessStatistics {
96    /// Total number of queries to this index
97    pub total_queries: u64,
98    /// Number of queries in the last hour
99    pub queries_last_hour: u64,
100    /// Number of queries in the last day
101    pub queries_last_day: u64,
102    /// Number of queries in the last week
103    pub queries_last_week: u64,
104    /// Average queries per second (moving average)
105    pub avg_qps: f64,
106    /// Peak queries per second
107    pub peak_qps: f64,
108    /// Last access timestamp
109    pub last_access_time: Option<SystemTime>,
110    /// Access pattern (hot, warm, cold)
111    pub access_pattern: AccessPattern,
112    /// Query latencies (p50, p95, p99 in microseconds)
113    pub query_latencies: LatencyPercentiles,
114}
115
116/// Access pattern classification
117#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
118pub enum AccessPattern {
119    /// Frequently accessed, high QPS
120    Hot,
121    /// Moderately accessed
122    Warm,
123    /// Rarely accessed
124    Cold,
125    /// Bursty access pattern
126    Bursty,
127    /// Seasonal pattern
128    Seasonal,
129    /// Unknown pattern (insufficient data)
130    Unknown,
131}
132
133impl Default for AccessPattern {
134    fn default() -> Self {
135        Self::Unknown
136    }
137}
138
139/// Latency percentiles
140#[derive(Debug, Clone, Default, Serialize, Deserialize)]
141pub struct LatencyPercentiles {
142    pub p50: u64,
143    pub p95: u64,
144    pub p99: u64,
145    pub max: u64,
146}
147
148/// Performance metrics for an index
149#[derive(Debug, Clone, Default, Serialize, Deserialize)]
150pub struct PerformanceMetrics {
151    /// Average query latency in microseconds
152    pub avg_query_latency_us: u64,
153    /// Average load time in milliseconds
154    pub avg_load_time_ms: u64,
155    /// Cache hit rate (0.0 - 1.0)
156    pub cache_hit_rate: f64,
157    /// Memory footprint in bytes
158    pub memory_footprint_bytes: u64,
159    /// CPU utilization (0.0 - 1.0)
160    pub cpu_utilization: f64,
161    /// IO operations per second
162    pub iops: f64,
163    /// Throughput in MB/s
164    pub throughput_mbps: f64,
165}
166
167/// Statistics for a storage tier
168#[derive(Debug, Clone, Serialize, Deserialize)]
169pub struct TierStatistics {
170    /// Total capacity in bytes
171    pub capacity_bytes: u64,
172    /// Used capacity in bytes
173    pub used_bytes: u64,
174    /// Number of indices in this tier
175    pub index_count: usize,
176    /// Total number of queries to this tier
177    pub total_queries: u64,
178    /// Average query latency in microseconds
179    pub avg_query_latency_us: u64,
180    /// Hit rate for this tier
181    pub hit_rate: f64,
182    /// Number of promotions from this tier
183    pub promotions: u64,
184    /// Number of demotions to this tier
185    pub demotions: u64,
186    /// Total bytes read
187    pub bytes_read: u64,
188    /// Total bytes written
189    pub bytes_written: u64,
190    /// Last update timestamp
191    pub last_updated: SystemTime,
192}
193
194impl Default for TierStatistics {
195    fn default() -> Self {
196        Self {
197            capacity_bytes: 0,
198            used_bytes: 0,
199            index_count: 0,
200            total_queries: 0,
201            avg_query_latency_us: 0,
202            hit_rate: 0.0,
203            promotions: 0,
204            demotions: 0,
205            bytes_read: 0,
206            bytes_written: 0,
207            last_updated: SystemTime::now(),
208        }
209    }
210}
211
212impl TierStatistics {
213    /// Calculate utilization ratio (0.0 - 1.0)
214    pub fn utilization(&self) -> f64 {
215        if self.capacity_bytes == 0 {
216            0.0
217        } else {
218            self.used_bytes as f64 / self.capacity_bytes as f64
219        }
220    }
221
222    /// Calculate available bytes
223    pub fn available_bytes(&self) -> u64 {
224        self.capacity_bytes.saturating_sub(self.used_bytes)
225    }
226
227    /// Check if tier is near capacity
228    pub fn is_near_capacity(&self, threshold: f64) -> bool {
229        self.utilization() >= threshold
230    }
231}
232
233/// Tier transition event
234#[derive(Debug, Clone, Serialize, Deserialize)]
235pub struct TierTransition {
236    /// Index identifier
237    pub index_id: String,
238    /// Source tier
239    pub from_tier: StorageTier,
240    /// Destination tier
241    pub to_tier: StorageTier,
242    /// Reason for transition
243    pub reason: String,
244    /// Timestamp of transition
245    pub timestamp: SystemTime,
246    /// Transition duration
247    pub duration: Duration,
248    /// Success flag
249    pub success: bool,
250    /// Error message (if failed)
251    pub error: Option<String>,
252}
253
254/// Configuration for gradual tier transitions
255#[derive(Debug, Clone, Serialize, Deserialize)]
256pub struct GradualTransitionConfig {
257    /// Enable gradual transitions
258    pub enabled: bool,
259    /// Number of stages for gradual transition
260    pub stages: usize,
261    /// Delay between stages
262    pub stage_delay: Duration,
263    /// Monitor performance during transition
264    pub monitor_performance: bool,
265    /// Rollback on performance degradation
266    pub rollback_on_degradation: bool,
267    /// Performance degradation threshold (0.0 - 1.0)
268    pub degradation_threshold: f64,
269}
270
271impl Default for GradualTransitionConfig {
272    fn default() -> Self {
273        Self {
274            enabled: true,
275            stages: 3,
276            stage_delay: Duration::from_secs(60),
277            monitor_performance: true,
278            rollback_on_degradation: true,
279            degradation_threshold: 0.2, // 20% degradation triggers rollback
280        }
281    }
282}
283
284/// Cost model for tier placement
285#[derive(Debug, Clone, Serialize, Deserialize)]
286pub struct TierCostModel {
287    /// Cost per GB per hour for hot tier
288    pub hot_cost_per_gb_hour: f64,
289    /// Cost per GB per hour for warm tier
290    pub warm_cost_per_gb_hour: f64,
291    /// Cost per GB per hour for cold tier
292    pub cold_cost_per_gb_hour: f64,
293    /// Cost per query for hot tier
294    pub hot_query_cost: f64,
295    /// Cost per query for warm tier
296    pub warm_query_cost: f64,
297    /// Cost per query for cold tier
298    pub cold_query_cost: f64,
299    /// Cost per tier transition
300    pub transition_cost: f64,
301}
302
303impl Default for TierCostModel {
304    fn default() -> Self {
305        Self {
306            hot_cost_per_gb_hour: 0.10,   // High cost for RAM
307            warm_cost_per_gb_hour: 0.02,  // Moderate cost for SSD
308            cold_cost_per_gb_hour: 0.005, // Low cost for HDD
309            hot_query_cost: 0.0001,       // Low per-query cost (fast)
310            warm_query_cost: 0.0005,      // Higher per-query cost (slower)
311            cold_query_cost: 0.002,       // Highest per-query cost (slowest)
312            transition_cost: 0.01,        // Cost of moving data between tiers
313        }
314    }
315}
316
317/// Resource allocation for multi-tenancy
318#[derive(Debug, Clone, Serialize, Deserialize)]
319pub struct TenantResourceAllocation {
320    /// Tenant identifier
321    pub tenant_id: String,
322    /// Allocated capacity in hot tier (bytes)
323    pub hot_tier_allocation_bytes: u64,
324    /// Allocated capacity in warm tier (bytes)
325    pub warm_tier_allocation_bytes: u64,
326    /// Allocated capacity in cold tier (bytes)
327    pub cold_tier_allocation_bytes: u64,
328    /// QPS quota
329    pub qps_quota: f64,
330    /// Priority (higher values = higher priority)
331    pub priority: u32,
332    /// Custom policies
333    pub custom_policies: HashMap<String, String>,
334}
oxirs_vec/tiering/types.rs

oxirs_vec/tiering/
types.rs
