1use crate::{
5 Utils,
6 config::normalize_grpc_endpoint,
7 constants,
8 models::ModelProvider,
9 providers::{
10 gcs::GcsProviderCache, huggingface::HuggingFaceProviderCache, ngc::NgcProviderCache,
11 },
12};
13use anyhow::{Context, Result};
14use serde::{Deserialize, Serialize};
15use std::env;
16use std::fs;
17use std::path::{Path, PathBuf};
18use tracing::{debug, info, warn};
19
20#[derive(Debug, Clone, Serialize, Deserialize)]
22pub struct CacheConfig {
23 pub local_path: PathBuf,
25 pub server_endpoint: String,
27 pub timeout_secs: Option<u64>,
29 #[serde(default = "default_shared_storage")]
32 pub shared_storage: bool,
33 #[serde(default = "default_transfer_chunk_size")]
35 pub transfer_chunk_size: usize,
36}
37
38fn default_shared_storage() -> bool {
39 constants::DEFAULT_SHARED_STORAGE
40}
41
42fn default_transfer_chunk_size() -> usize {
43 constants::DEFAULT_TRANSFER_CHUNK_SIZE
44}
45
46impl Default for CacheConfig {
47 fn default() -> Self {
48 let home = Utils::get_home_dir().unwrap_or_else(|_| ".".to_string());
49 Self {
50 local_path: PathBuf::from(home).join(constants::DEFAULT_CACHE_PATH),
51 server_endpoint: format!("http://localhost:{}", constants::DEFAULT_GRPC_PORT),
52 timeout_secs: None,
53 shared_storage: constants::DEFAULT_SHARED_STORAGE,
54 transfer_chunk_size: constants::DEFAULT_TRANSFER_CHUNK_SIZE,
55 }
56 }
57}
58
59impl CacheConfig {
60 pub fn discover() -> Result<Self> {
62 if let Some(path) = Self::get_cache_path_from_args() {
71 return Self::from_path(path);
72 }
73
74 if let Some(path) = crate::envs::cache_directory() {
76 return Self::from_path(path);
77 }
78
79 if let Ok(config) = Self::from_config_file() {
81 return Ok(config);
82 }
83
84 if let Ok(config) = Self::auto_detect() {
86 return Ok(config);
87 }
88
89 debug!("Using default cache configuration");
91 Ok(Self::default())
92 }
93
94 pub fn new(local_path: PathBuf, server_endpoint: Option<String>) -> Result<Self> {
96 fs::create_dir_all(&local_path)
98 .with_context(|| format!("Failed to create cache directory: {local_path:?}"))?;
99
100 Ok(Self {
101 local_path,
102 server_endpoint: normalize_grpc_endpoint(
103 server_endpoint.unwrap_or_else(Self::get_default_server_endpoint),
104 ),
105 timeout_secs: None,
106 shared_storage: constants::DEFAULT_SHARED_STORAGE,
107 transfer_chunk_size: constants::DEFAULT_TRANSFER_CHUNK_SIZE,
108 })
109 }
110
111 pub fn from_path<P: AsRef<Path>>(path: P) -> Result<Self> {
113 let local_path = path.as_ref().to_path_buf();
114
115 fs::create_dir_all(&local_path)
117 .with_context(|| format!("Failed to create cache directory: {local_path:?}"))?;
118
119 Ok(Self {
120 local_path,
121 server_endpoint: Self::get_default_server_endpoint(),
122 timeout_secs: None,
123 shared_storage: constants::DEFAULT_SHARED_STORAGE,
124 transfer_chunk_size: constants::DEFAULT_TRANSFER_CHUNK_SIZE,
125 })
126 }
127
128 pub fn from_config_file() -> Result<Self> {
130 let config_path = Self::get_config_path()?;
131
132 if !config_path.exists() {
133 return Err(anyhow::anyhow!("Config file not found: {:?}", config_path));
134 }
135
136 let content = fs::read_to_string(&config_path)
137 .with_context(|| format!("Failed to read config file: {config_path:?}"))?;
138
139 let mut config: Self = serde_yaml::from_str(&content)
140 .with_context(|| format!("Failed to parse config file: {config_path:?}"))?;
141 config.server_endpoint =
142 normalize_grpc_endpoint(std::mem::take(&mut config.server_endpoint));
143
144 Ok(config)
145 }
146
147 pub fn save_to_config_file(&self) -> Result<()> {
149 let config_path = Self::get_config_path()?;
150
151 if let Some(parent) = config_path.parent() {
153 fs::create_dir_all(parent)
154 .with_context(|| format!("Failed to create config directory: {parent:?}"))?;
155 }
156
157 let content = serde_yaml::to_string(self).context("Failed to serialize config")?;
158
159 fs::write(&config_path, content)
160 .with_context(|| format!("Failed to write config file: {config_path:?}"))?;
161
162 Ok(())
163 }
164
165 pub fn auto_detect() -> Result<Self> {
167 let home = Utils::get_home_dir().unwrap_or_else(|_| ".".to_string());
168 let common_paths = vec![
169 PathBuf::from(&home).join(constants::DEFAULT_CACHE_PATH),
170 PathBuf::from(&home).join(constants::DEFAULT_HF_CACHE_PATH),
171 PathBuf::from("/cache"),
172 PathBuf::from("/app/models"),
173 PathBuf::from("./cache"),
174 PathBuf::from("./models"),
175 ];
176
177 for path in common_paths {
178 if path.exists() && path.is_dir() {
179 return Ok(Self {
180 local_path: path,
181 server_endpoint: Self::get_default_server_endpoint(),
182 timeout_secs: None,
183 shared_storage: constants::DEFAULT_SHARED_STORAGE,
184 transfer_chunk_size: constants::DEFAULT_TRANSFER_CHUNK_SIZE,
185 });
186 }
187 }
188
189 Err(anyhow::anyhow!(
190 "No cache directory found in common locations"
191 ))
192 }
193
194 pub fn from_server() -> Result<Self> {
196 Err(anyhow::anyhow!("Server not available for cache discovery"))
199 }
200
201 fn get_cache_path_from_args() -> Option<String> {
203 let args: Vec<String> = env::args().collect();
204
205 for (i, arg) in args.iter().enumerate() {
206 if arg == "--cache-path"
207 && let Some(next_arg) = args.get(i.saturating_add(1))
208 {
209 return Some(next_arg.clone());
210 }
211 }
212
213 None
214 }
215
216 fn get_default_server_endpoint() -> String {
218 normalize_grpc_endpoint(crate::envs::server_endpoint_or_default())
219 }
220
221 fn get_config_path() -> Result<PathBuf> {
223 let home = Utils::get_home_dir().unwrap_or_else(|_| ".".to_string());
224
225 Ok(PathBuf::from(home).join(constants::DEFAULT_CONFIG_PATH))
226 }
227
228 pub fn get_cache_stats(&self) -> Result<CacheStats> {
230 let mut models = Vec::new();
231
232 if !self.local_path.exists() {
233 return Ok(CacheStats {
234 total_models: 0,
235 total_size: 0,
236 models,
237 });
238 }
239
240 for provider in [
241 ModelProvider::HuggingFace,
242 ModelProvider::Ngc,
243 ModelProvider::Gcs,
244 ] {
245 models.extend(cache_for_provider(provider).list_models(&self.local_path)?);
246 }
247
248 models.sort_by(|left, right| {
249 provider_sort_key(left.provider)
250 .cmp(&provider_sort_key(right.provider))
251 .then_with(|| left.name.cmp(&right.name))
252 });
253
254 let total_size = models.iter().map(|model| model.size).sum();
255
256 Ok(CacheStats {
257 total_models: models.len(),
258 total_size,
259 models,
260 })
261 }
262
263 pub fn clear_model(&self, model_name: &str, provider: ModelProvider) -> Result<()> {
265 cache_for_provider(provider).clear_model(&self.local_path, model_name)
266 }
267
268 pub fn clear_all(&self) -> Result<()> {
270 if self.local_path.exists() {
271 for entry in fs::read_dir(&self.local_path)
272 .with_context(|| format!("Failed to read cache directory: {:?}", self.local_path))?
273 {
274 let entry = entry
275 .with_context(|| format!("Failed to read entry in: {:?}", self.local_path))?;
276 let path = entry.path();
277 if path.is_dir() {
278 fs::remove_dir_all(&path)
279 .with_context(|| format!("Failed to remove directory: {:?}", path))?;
280 } else {
281 fs::remove_file(&path)
282 .with_context(|| format!("Failed to remove file: {:?}", path))?;
283 }
284 }
285 info!("Cleared entire cache");
286 } else {
287 warn!("Cache directory does not exist");
288 }
289
290 Ok(())
291 }
292}
293
294#[derive(Debug, Clone)]
296pub struct CacheStats {
297 pub total_models: usize,
298 pub total_size: u64,
299 pub models: Vec<ModelInfo>,
300}
301
302#[derive(Debug, Clone)]
304pub struct ModelInfo {
305 pub provider: ModelProvider,
306 pub name: String,
307 pub size: u64,
308 pub path: PathBuf,
309}
310
311impl CacheStats {
312 fn format_bytes(bytes: u64) -> String {
314 const KB: u64 = 1024;
315 const MB: u64 = KB * 1024;
316 const GB: u64 = MB * 1024;
317
318 match bytes {
319 size if size >= GB => format!("{:.2} GB", size as f64 / GB as f64),
320 size if size >= MB => format!("{:.2} MB", size as f64 / MB as f64),
321 size if size >= KB => format!("{:.2} KB", size as f64 / KB as f64),
322 size => format!("{size} bytes"),
323 }
324 }
325
326 pub fn format_total_size(&self) -> String {
328 Self::format_bytes(self.total_size)
329 }
330
331 pub fn format_model_size(&self, model: &ModelInfo) -> String {
333 Self::format_bytes(model.size)
334 }
335}
336
337pub(crate) trait ProviderCache: Send + Sync {
338 fn clear_model(&self, cache_root: &Path, model_name: &str) -> Result<()>;
339 fn resolve_model_path(
340 &self,
341 cache_root: &Path,
342 model_name: &str,
343 revision: Option<&str>,
344 ) -> Result<PathBuf>;
345 fn list_models(&self, cache_root: &Path) -> Result<Vec<ModelInfo>>;
346}
347
348pub(crate) fn cache_for_provider(provider: ModelProvider) -> &'static dyn ProviderCache {
349 match provider {
350 ModelProvider::HuggingFace => &HuggingFaceProviderCache,
351 ModelProvider::Ngc => &NgcProviderCache,
352 ModelProvider::Gcs => &GcsProviderCache,
353 }
354}
355
356pub fn resolve_model_path(
357 cache_root: &Path,
358 provider: ModelProvider,
359 model_name: &str,
360 revision: Option<&str>,
361) -> Result<PathBuf> {
362 cache_for_provider(provider).resolve_model_path(cache_root, model_name, revision)
363}
364
365pub(crate) fn directory_size(path: &Path) -> Result<u64> {
366 let mut size: u64 = 0;
367
368 for entry in fs::read_dir(path)? {
369 let entry = entry?;
370 let path = entry.path();
371
372 if path.is_file() {
373 size = size.saturating_add(fs::metadata(&path)?.len());
374 } else if path.is_dir() {
375 size = size.saturating_add(directory_size(&path)?);
376 }
377 }
378
379 Ok(size)
380}
381
382fn provider_sort_key(provider: ModelProvider) -> u8 {
383 match provider {
384 ModelProvider::HuggingFace => 0,
385 ModelProvider::Ngc => 1,
386 ModelProvider::Gcs => 2,
387 }
388}
389
390#[cfg(test)]
391#[allow(clippy::expect_used)]
392mod tests {
393 use super::*;
394 use crate::Utils;
395 use tempfile::TempDir;
396
397 #[test]
398 #[allow(clippy::expect_used)]
399 fn test_cache_config_from_path() {
400 let temp_dir = TempDir::new().expect("Failed to create temp directory");
401 let config =
402 CacheConfig::from_path(temp_dir.path()).expect("Failed to create config from path");
403
404 assert_eq!(config.local_path, temp_dir.path());
405 }
406
407 #[test]
408 #[allow(clippy::expect_used)]
409 fn test_cache_config_save_and_load() {
410 let temp_dir = TempDir::new().expect("Failed to create temp directory");
411 let original_config = CacheConfig {
412 local_path: temp_dir.path().join("cache"),
413 server_endpoint: "http://localhost:8001".to_string(),
414 timeout_secs: Some(30),
415 shared_storage: false,
416 transfer_chunk_size: 64 * 1024,
417 };
418
419 original_config
421 .save_to_config_file()
422 .expect("Failed to save config");
423
424 let loaded_config = CacheConfig::from_config_file().expect("Failed to load config");
426
427 assert_eq!(loaded_config.local_path, original_config.local_path);
428 assert_eq!(
429 loaded_config.server_endpoint,
430 original_config.server_endpoint
431 );
432 assert_eq!(loaded_config.timeout_secs, original_config.timeout_secs);
433 assert_eq!(loaded_config.shared_storage, original_config.shared_storage);
434 assert_eq!(
435 loaded_config.transfer_chunk_size,
436 original_config.transfer_chunk_size
437 );
438 }
439
440 #[test]
441 fn test_cache_stats_formatting() {
442 let stats = CacheStats {
443 total_models: 2,
444 total_size: 1024 * 1024 * 5, models: vec![
446 ModelInfo {
447 provider: ModelProvider::HuggingFace,
448 name: "model1".to_string(),
449 size: 1024 * 1024 * 2, path: PathBuf::from("/test/model1"),
451 },
452 ModelInfo {
453 provider: ModelProvider::Gcs,
454 name: "gs://bucket/model2".to_string(),
455 size: 1024 * 1024 * 3, path: PathBuf::from("/test/model2"),
457 },
458 ],
459 };
460
461 assert_eq!(stats.format_total_size(), "5.00 MB");
462 assert_eq!(stats.format_model_size(&stats.models[0]), "2.00 MB");
463 assert_eq!(stats.format_model_size(&stats.models[1]), "3.00 MB");
464 }
465
466 #[test]
467 fn test_cache_config_default() {
468 let config = CacheConfig::default();
469
470 let home = Utils::get_home_dir().unwrap_or_else(|_| ".".to_string());
471 assert_eq!(
472 config.local_path,
473 PathBuf::from(&home).join(constants::DEFAULT_CACHE_PATH)
474 );
475 assert_eq!(
476 config.server_endpoint,
477 String::from("http://localhost:8001")
478 );
479 assert_eq!(config.timeout_secs, None);
480 assert!(config.shared_storage);
481 assert_eq!(
482 config.transfer_chunk_size,
483 constants::DEFAULT_TRANSFER_CHUNK_SIZE
484 );
485 }
486
487 #[test]
488 #[allow(clippy::expect_used)]
489 fn test_cache_config_new_accepts_bare_host_port() {
490 let temp_dir = TempDir::new().expect("Failed to create temp directory");
491 let config = CacheConfig::new(
492 temp_dir.path().join("cache"),
493 Some("modelexpress-server:8001".to_string()),
494 )
495 .expect("Failed to create cache config");
496
497 assert_eq!(config.server_endpoint, "http://modelexpress-server:8001");
498 }
499
500 #[test]
501 #[allow(clippy::expect_used)]
502 fn test_get_config_path() {
503 let config_path = CacheConfig::get_config_path().expect("Failed to get config path");
504
505 let home = Utils::get_home_dir().unwrap_or_else(|_| ".".to_string());
506 assert_eq!(
507 config_path,
508 PathBuf::from(&home).join(constants::DEFAULT_CONFIG_PATH)
509 );
510 }
511
512 #[test]
513 fn test_resolve_model_path_huggingface_uses_snapshot_layout() {
514 let cache_root = Path::new("/tmp/cache");
515
516 assert_eq!(
517 resolve_model_path(
518 cache_root,
519 ModelProvider::HuggingFace,
520 "google/t5-small",
521 Some("abc123"),
522 )
523 .expect("Expected HF model path"),
524 PathBuf::from("/tmp/cache/models--google--t5-small/snapshots/abc123")
525 );
526 }
527
528 #[test]
529 fn test_resolve_model_path_gcs_uses_full_url_layout() {
530 let cache_root = Path::new("/tmp/cache");
531
532 assert_eq!(
533 resolve_model_path(
534 cache_root,
535 ModelProvider::Gcs,
536 "gs://envbucket/dev/bake/qwen/rev123",
537 None,
538 )
539 .expect("Expected GCS model path"),
540 PathBuf::from("/tmp/cache/gcs/envbucket/dev/bake/qwen/rev123")
541 );
542 }
543
544 #[test]
545 fn test_resolve_model_path_gcs_full_url_trailing_slash_normalizes() {
546 let cache_root = Path::new("/tmp/cache");
547
548 assert_eq!(
549 resolve_model_path(
550 cache_root,
551 ModelProvider::Gcs,
552 "gs://sourcebucket/dev/bake/qwen/rev123/",
553 None,
554 )
555 .expect("Expected GCS model path"),
556 PathBuf::from("/tmp/cache/gcs/sourcebucket/dev/bake/qwen/rev123")
557 );
558 }
559
560 fn create_test_cache_config(local_path: PathBuf) -> CacheConfig {
561 CacheConfig {
562 local_path,
563 server_endpoint: "http://localhost:8001".to_string(),
564 timeout_secs: None,
565 shared_storage: false,
566 transfer_chunk_size: 64 * 1024,
567 }
568 }
569
570 #[test]
571 fn test_get_cache_stats_supports_hf_and_gcs_layouts() {
572 let temp_dir = TempDir::new().expect("Failed to create temp directory");
573 let cache_path = temp_dir.path().join("cache");
574 fs::create_dir_all(&cache_path).expect("Failed to create cache directory");
575
576 let hf_model_dir = cache_path.join("models--google--t5-small");
577 fs::create_dir_all(&hf_model_dir).expect("Failed to create HF model directory");
578 fs::write(hf_model_dir.join("config.json"), b"{}").expect("Failed to write HF file");
579
580 let gcs_model_dir = resolve_model_path(
581 &cache_path,
582 ModelProvider::Gcs,
583 "gs://envbucket/dev/bake/qwen/rev123",
584 None,
585 )
586 .expect("Failed to resolve GCS path");
587 fs::create_dir_all(gcs_model_dir.join("weights"))
588 .expect("Failed to create GCS model directory");
589 fs::write(gcs_model_dir.join("tokenizer.json"), b"{}")
590 .expect("Failed to write GCS tokenizer");
591 fs::write(gcs_model_dir.join("weights/model.bin"), b"abcd")
592 .expect("Failed to write GCS weights");
593 let gcs_metadata_dir = gcs_model_dir.join(".mx");
594 fs::create_dir_all(&gcs_metadata_dir).expect("Failed to create GCS metadata directory");
595 fs::write(
596 gcs_metadata_dir.join("manifest.json"),
597 r#"{"version":1,"model":"gs://envbucket/dev/bake/qwen/rev123","files":[{"path":"tokenizer.json","size":2,"crc32c":"00000000","generation":null},{"path":"weights/model.bin","size":4,"crc32c":"00000000","generation":null}]}
598"#,
599 )
600 .expect("Failed to write GCS manifest");
601
602 let ignored_dir = cache_path.join("tmp");
603 fs::create_dir_all(&ignored_dir).expect("Failed to create ignored directory");
604 fs::write(ignored_dir.join("scratch.txt"), b"ignore")
605 .expect("Failed to write ignored file");
606
607 let stats = create_test_cache_config(cache_path)
608 .get_cache_stats()
609 .expect("Failed to get cache stats");
610
611 assert_eq!(stats.total_models, 2);
612 assert_eq!(stats.total_size, 8);
613 assert_eq!(stats.models.len(), 2);
614
615 assert_eq!(stats.models[0].provider, ModelProvider::HuggingFace);
616 assert_eq!(stats.models[0].name, "google/t5-small");
617 assert_eq!(stats.models[0].size, 2);
618 assert_eq!(stats.models[0].path, hf_model_dir);
619 assert_eq!(stats.models[1].provider, ModelProvider::Gcs);
620 assert_eq!(stats.models[1].name, "gs://envbucket/dev/bake/qwen/rev123");
621 assert_eq!(stats.models[1].size, 6);
622 assert_eq!(stats.models[1].path, gcs_model_dir);
623 assert!(stats.models.iter().all(|model| model.name != "tmp"));
624 }
625
626 #[test]
627 fn test_clear_model_removes_only_requested_layout() {
628 let temp_dir = TempDir::new().expect("Failed to create temp directory");
629 let cache_path = temp_dir.path().join("cache");
630 fs::create_dir_all(&cache_path).expect("Failed to create cache directory");
631
632 let hf_model_dir = cache_path.join("models--google--t5-small");
633 fs::create_dir_all(&hf_model_dir).expect("Failed to create HF model directory");
634 fs::write(hf_model_dir.join("config.json"), b"{}").expect("Failed to write HF file");
635
636 let gcs_model_dir = resolve_model_path(
637 &cache_path,
638 ModelProvider::Gcs,
639 "gs://envbucket/org/model/rev1",
640 None,
641 )
642 .expect("Failed to resolve GCS path");
643 fs::create_dir_all(&gcs_model_dir).expect("Failed to create GCS model directory");
644 fs::write(gcs_model_dir.join("tokenizer.json"), b"{}").expect("Failed to write GCS file");
645
646 let config = create_test_cache_config(cache_path);
647
648 config
649 .clear_model("gs://envbucket/org/model/rev1", ModelProvider::Gcs)
650 .expect("Failed to clear GCS model");
651 assert!(hf_model_dir.exists(), "HF model should remain");
652 assert!(!gcs_model_dir.exists(), "GCS model should be removed");
653
654 config
655 .clear_model("google/t5-small", ModelProvider::HuggingFace)
656 .expect("Failed to clear HF model");
657 assert!(!hf_model_dir.exists(), "HF model should be removed");
658 }
659
660 #[test]
661 fn test_clear_all_removes_contents_but_keeps_directory() {
662 let temp_dir = TempDir::new().expect("Failed to create temp directory");
663 let cache_path = temp_dir.path().join("cache");
664 fs::create_dir_all(&cache_path).expect("Failed to create cache directory");
665
666 let model_dir = cache_path.join("models--test--model");
668 fs::create_dir_all(&model_dir).expect("Failed to create model directory");
669 fs::write(model_dir.join("config.json"), "{}").expect("Failed to write file");
670 fs::write(cache_path.join("test_file.txt"), "test").expect("Failed to write file");
671
672 let config = create_test_cache_config(cache_path.clone());
673
674 config.clear_all().expect("Failed to clear cache");
676
677 assert!(cache_path.exists(), "Cache directory should still exist");
679 assert!(
680 fs::read_dir(&cache_path)
681 .expect("Failed to read dir")
682 .next()
683 .is_none(),
684 "Cache directory should be empty"
685 );
686 }
687
688 #[test]
689 fn test_clear_all_handles_nonexistent_directory() {
690 let temp_dir = TempDir::new().expect("Failed to create temp directory");
691 let cache_path = temp_dir.path().join("nonexistent_cache");
692
693 let config = create_test_cache_config(cache_path.clone());
694
695 config
697 .clear_all()
698 .with_context(|| format!("Failed to clear cache: {cache_path:?}"))
699 .expect("Failed to clear cache");
700 assert!(!cache_path.exists());
701 }
702
703 #[test]
704 fn test_clear_all_removes_nested_directories() {
705 let temp_dir = TempDir::new().expect("Failed to create temp directory");
706 let cache_path = temp_dir.path().join("cache");
707 fs::create_dir_all(&cache_path).expect("Failed to create cache directory");
708
709 let deep_path = cache_path.join("a").join("b").join("c");
711 fs::create_dir_all(&deep_path).expect("Failed to create nested directories");
712 fs::write(deep_path.join("deep_file.txt"), "deep").expect("Failed to write file");
713
714 let config = create_test_cache_config(cache_path.clone());
715
716 config.clear_all().expect("Failed to clear cache");
717
718 assert!(cache_path.exists(), "Cache directory should still exist");
719 assert!(
720 fs::read_dir(&cache_path)
721 .expect("Failed to read dir")
722 .next()
723 .is_none(),
724 "Cache directory should be empty after clearing nested content"
725 );
726 }
727}