selfware 0.2.2

Your personal AI workshop — software you own, software that lasts
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
//! Resource management for CPU, GPU, memory, and disk

use crate::config::ResourcesConfig;
use crate::errors::{ResourceError, SelfwareError};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;
use tracing::{debug, info, warn};

pub mod disk;
pub mod gpu;
pub mod memory;
pub mod quotas;

pub use disk::DiskManager;
pub use gpu::GpuManager;
pub use memory::MemoryManager;
pub use quotas::AdaptiveQuotas;

/// Resource manager for coordinating all resource types
pub struct ResourceManager {
    config: ResourcesConfig,
    gpu: Arc<GpuManager>,
    memory: Arc<MemoryManager>,
    disk: Arc<DiskManager>,
    quotas: Arc<RwLock<AdaptiveQuotas>>,
    usage: Arc<RwLock<ResourceUsage>>,
    shared_pressure: Arc<std::sync::RwLock<ResourcePressure>>,
}

/// Current resource usage
#[derive(Debug, Clone, Default)]
pub struct ResourceUsage {
    pub cpu_percent: f32,
    pub memory_used_bytes: u64,
    pub memory_total_bytes: u64,
    pub gpu_memory_used_bytes: u64,
    pub gpu_memory_total_bytes: u64,
    pub gpu_utilization: f32,
    pub gpu_temperature: u32,
    pub disk_used_bytes: u64,
    pub disk_total_bytes: u64,
}

/// Resource pressure levels
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResourcePressure {
    None,
    Low,
    Medium,
    High,
    Critical,
}

impl ResourcePressure {
    /// Check if pressure is critical
    pub fn is_critical(&self) -> bool {
        matches!(self, Self::Critical)
    }

    /// Check if pressure requires action
    pub fn requires_action(&self) -> bool {
        matches!(self, Self::Medium | Self::High | Self::Critical)
    }
}

impl ResourceManager {
    /// Create a new resource manager
    pub async fn new(config: &ResourcesConfig) -> Result<Self, SelfwareError> {
        let gpu = Arc::new(GpuManager::new(&config.gpu).await?);
        let memory = Arc::new(MemoryManager::new(&config.memory).await?);
        let disk = Arc::new(DiskManager::new(&config.disk).await?);

        let quotas = Arc::new(RwLock::new(AdaptiveQuotas::new(config.quotas.clone())));
        let usage = Arc::new(RwLock::new(ResourceUsage::default()));

        Ok(Self {
            config: config.clone(),
            gpu,
            memory,
            disk,
            quotas,
            usage,
            shared_pressure: Arc::new(std::sync::RwLock::new(ResourcePressure::None)),
        })
    }

    /// Start resource monitoring loop
    pub async fn monitor_loop(&self) {
        let mut interval = tokio::time::interval(Duration::from_secs(5));

        loop {
            interval.tick().await;

            // Update resource usage
            self.update_usage().await;

            // Check resource pressure
            let pressure = self.get_resource_pressure().await;

            // Publish to shared handle so other subsystems (e.g. Swarm) can read it.
            *self
                .shared_pressure
                .write()
                .unwrap_or_else(|e| e.into_inner()) = pressure;

            if pressure.requires_action() {
                warn!(pressure = ?pressure, "Resource pressure detected");
                self.handle_pressure(pressure).await;
            }

            // Update adaptive quotas
            {
                let quotas = self.quotas.write().await;
                quotas.adjust_for_pressure(pressure).await;
            }
        }
    }

    /// Update current resource usage
    async fn update_usage(&self) {
        let mut usage = self.usage.write().await;

        // Get GPU usage
        if let Ok(gpu_usage) = self.gpu.get_usage().await {
            usage.gpu_memory_used_bytes = gpu_usage.memory_used;
            usage.gpu_memory_total_bytes = gpu_usage.memory_total;
            usage.gpu_utilization = gpu_usage.utilization;
            usage.gpu_temperature = gpu_usage.temperature;
        }

        // Get memory usage
        if let Ok(mem_usage) = self.memory.get_usage().await {
            usage.memory_used_bytes = mem_usage.used;
            usage.memory_total_bytes = mem_usage.total;
        }

        // Get disk usage
        if let Ok(disk_usage) = self.disk.get_usage().await {
            usage.disk_used_bytes = disk_usage.used;
            usage.disk_total_bytes = disk_usage.total;
        }

        // Emit metrics
        // metrics::gauge!("resource.memory.used_bytes", usage.memory_used_bytes as f64);
        // metrics::gauge!("resource.gpu.memory.used_bytes", usage.gpu_memory_used_bytes as f64);
        // metrics::gauge!("resource.gpu.temperature", usage.gpu_temperature as f64);
        // metrics::gauge!("resource.disk.used_bytes", usage.disk_used_bytes as f64);
    }

    /// Get current resource pressure
    pub async fn get_resource_pressure(&self) -> ResourcePressure {
        let usage = self.usage.read().await;

        let memory_ratio = usage.memory_used_bytes as f32 / usage.memory_total_bytes as f32;
        let gpu_memory_ratio = if usage.gpu_memory_total_bytes > 0 {
            usage.gpu_memory_used_bytes as f32 / usage.gpu_memory_total_bytes as f32
        } else {
            0.0
        };

        // Determine overall pressure
        let max_ratio = memory_ratio.max(gpu_memory_ratio);

        if max_ratio > self.config.memory.emergency_threshold {
            ResourcePressure::Critical
        } else if max_ratio > self.config.memory.critical_threshold {
            ResourcePressure::High
        } else if max_ratio > self.config.memory.warning_threshold {
            ResourcePressure::Medium
        } else if max_ratio > 0.5 {
            ResourcePressure::Low
        } else {
            ResourcePressure::None
        }
    }

    /// Handle resource pressure
    async fn handle_pressure(&self, pressure: ResourcePressure) {
        match pressure {
            ResourcePressure::Critical => {
                // Emergency measures
                self.memory.trigger_emergency_cleanup().await;
                self.gpu.throttle_compute(0.5).await;
            }
            ResourcePressure::High => {
                // Aggressive cleanup
                self.memory.trigger_critical_cleanup().await;
                self.gpu.reduce_batch_size().await;
            }
            ResourcePressure::Medium => {
                // Moderate cleanup
                self.memory.trigger_warning_cleanup().await;
            }
            _ => {}
        }
    }

    /// Get a shared handle to the current resource pressure, suitable for
    /// passing to other subsystems (e.g. `Swarm::set_resource_pressure`).
    pub fn shared_pressure(&self) -> Arc<std::sync::RwLock<ResourcePressure>> {
        Arc::clone(&self.shared_pressure)
    }

    /// Get current resource usage
    pub async fn get_usage(&self) -> ResourceUsage {
        self.usage.read().await.clone()
    }

    /// Report metrics
    pub async fn report_metrics(&self) -> Result<(), SelfwareError> {
        let usage = self.get_usage().await;

        info!(
            memory_used_gb = usage.memory_used_bytes / 1_000_000_000,
            gpu_memory_used_gb = usage.gpu_memory_used_bytes / 1_000_000_000,
            gpu_temp = usage.gpu_temperature,
            disk_used_gb = usage.disk_used_bytes / 1_000_000_000,
            "Resource usage report"
        );

        Ok(())
    }

    /// Check if operation is within quotas
    pub async fn check_quotas(&self, required: &ResourceRequest) -> Result<(), ResourceError> {
        let quotas = self.quotas.read().await;
        quotas.check(required).await
    }

    /// Reserve resources for an operation
    pub async fn reserve(
        &self,
        request: ResourceRequest,
    ) -> Result<ResourceReservation, ResourceError> {
        self.check_quotas(&request).await?;

        Ok(ResourceReservation {
            request,
            reserved_at: std::time::Instant::now(),
        })
    }
}

/// Resource request
#[derive(Debug, Clone)]
pub struct ResourceRequest {
    pub gpu_memory_bytes: u64,
    pub system_memory_bytes: u64,
    pub disk_bytes: u64,
    pub duration_estimate: Duration,
}

/// Resource reservation
#[derive(Debug, Clone)]
pub struct ResourceReservation {
    pub request: ResourceRequest,
    pub reserved_at: std::time::Instant,
}

impl ResourceReservation {
    /// Release the reservation
    pub fn release(self) {
        // In a real implementation, this would update resource tracking
        debug!("Resource reservation released");
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // ---- ResourcePressure tests ----

    #[test]
    fn test_resource_pressure_is_critical_only_for_critical() {
        assert!(ResourcePressure::Critical.is_critical());
        assert!(!ResourcePressure::High.is_critical());
        assert!(!ResourcePressure::Medium.is_critical());
        assert!(!ResourcePressure::Low.is_critical());
        assert!(!ResourcePressure::None.is_critical());
    }

    #[test]
    fn test_resource_pressure_requires_action() {
        assert!(ResourcePressure::Critical.requires_action());
        assert!(ResourcePressure::High.requires_action());
        assert!(ResourcePressure::Medium.requires_action());
        assert!(!ResourcePressure::Low.requires_action());
        assert!(!ResourcePressure::None.requires_action());
    }

    #[test]
    fn test_resource_pressure_equality() {
        assert_eq!(ResourcePressure::None, ResourcePressure::None);
        assert_eq!(ResourcePressure::Critical, ResourcePressure::Critical);
        assert_ne!(ResourcePressure::Low, ResourcePressure::High);
    }

    #[test]
    fn test_resource_pressure_clone() {
        let p = ResourcePressure::High;
        let p2 = p;
        assert_eq!(p, p2);
    }

    // ---- ResourceUsage tests ----

    #[test]
    fn test_resource_usage_default() {
        let usage = ResourceUsage::default();
        assert_eq!(usage.cpu_percent, 0.0);
        assert_eq!(usage.memory_used_bytes, 0);
        assert_eq!(usage.memory_total_bytes, 0);
        assert_eq!(usage.gpu_memory_used_bytes, 0);
        assert_eq!(usage.gpu_memory_total_bytes, 0);
        assert_eq!(usage.gpu_utilization, 0.0);
        assert_eq!(usage.gpu_temperature, 0);
        assert_eq!(usage.disk_used_bytes, 0);
        assert_eq!(usage.disk_total_bytes, 0);
    }

    #[test]
    fn test_resource_usage_clone() {
        let usage = ResourceUsage {
            cpu_percent: 55.0,
            memory_used_bytes: 8_000_000_000,
            memory_total_bytes: 16_000_000_000,
            gpu_memory_used_bytes: 4_000_000_000,
            gpu_memory_total_bytes: 24_000_000_000,
            gpu_utilization: 80.0,
            gpu_temperature: 72,
            disk_used_bytes: 500_000_000_000,
            disk_total_bytes: 1_000_000_000_000,
        };
        let cloned = usage.clone();
        assert_eq!(cloned.cpu_percent, 55.0);
        assert_eq!(cloned.memory_used_bytes, 8_000_000_000);
        assert_eq!(cloned.gpu_temperature, 72);
    }

    // ---- ResourceRequest tests ----

    #[test]
    fn test_resource_request_creation() {
        let req = ResourceRequest {
            gpu_memory_bytes: 10_000_000_000,
            system_memory_bytes: 4_000_000_000,
            disk_bytes: 1_000_000_000,
            duration_estimate: Duration::from_secs(300),
        };
        assert_eq!(req.gpu_memory_bytes, 10_000_000_000);
        assert_eq!(req.system_memory_bytes, 4_000_000_000);
        assert_eq!(req.disk_bytes, 1_000_000_000);
        assert_eq!(req.duration_estimate, Duration::from_secs(300));
    }

    #[test]
    fn test_resource_request_clone() {
        let req = ResourceRequest {
            gpu_memory_bytes: 5_000_000_000,
            system_memory_bytes: 2_000_000_000,
            disk_bytes: 500_000_000,
            duration_estimate: Duration::from_secs(60),
        };
        let cloned = req.clone();
        assert_eq!(cloned.gpu_memory_bytes, req.gpu_memory_bytes);
        assert_eq!(cloned.duration_estimate, req.duration_estimate);
    }

    // ---- ResourceReservation tests ----

    #[test]
    fn test_resource_reservation_creation_and_release() {
        let req = ResourceRequest {
            gpu_memory_bytes: 1_000_000,
            system_memory_bytes: 500_000,
            disk_bytes: 100_000,
            duration_estimate: Duration::from_secs(10),
        };
        let reservation = ResourceReservation {
            request: req,
            reserved_at: std::time::Instant::now(),
        };
        assert_eq!(reservation.request.gpu_memory_bytes, 1_000_000);
        // release consumes self; just verify it doesn't panic
        reservation.release();
    }

    #[test]
    fn test_resource_reservation_tracks_time() {
        let before = std::time::Instant::now();
        let req = ResourceRequest {
            gpu_memory_bytes: 0,
            system_memory_bytes: 0,
            disk_bytes: 0,
            duration_estimate: Duration::from_secs(0),
        };
        let reservation = ResourceReservation {
            request: req,
            reserved_at: std::time::Instant::now(),
        };
        let after = std::time::Instant::now();
        assert!(reservation.reserved_at >= before);
        assert!(reservation.reserved_at <= after);
    }

    #[test]
    fn test_resource_reservation_clone() {
        let req = ResourceRequest {
            gpu_memory_bytes: 42,
            system_memory_bytes: 99,
            disk_bytes: 7,
            duration_estimate: Duration::from_millis(500),
        };
        let reservation = ResourceReservation {
            request: req,
            reserved_at: std::time::Instant::now(),
        };
        let cloned = reservation.clone();
        assert_eq!(
            cloned.request.gpu_memory_bytes,
            reservation.request.gpu_memory_bytes
        );
    }
}