foundry-local 0.1.0

SDK for Microsoft Foundry Local service
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

#[cfg(feature = "integration-tests")]
mod integration_tests {
    use foundry_local::FoundryLocalManager;

    async fn setup_manager() -> FoundryLocalManager {
        FoundryLocalManager::builder()
            .bootstrap(true)
            .build()
            .await
            .expect("Failed to create manager")
    }

    #[tokio::test]
    async fn test_real_service_workflow() {
        // Create manager
        let mut manager = setup_manager().await;

        // List catalog models and get the first model ID
        let model_id = {
            let catalog_models = manager
                .list_catalog_models()
                .await
                .expect("Failed to list catalog models");

            assert!(!catalog_models.is_empty(), "No models found in catalog");

            let first_model = &catalog_models[0];
            println!(
                "Testing with model: {} ({})",
                first_model.alias, first_model.id
            );
            first_model.id.clone()
        };

        // Download the model
        let model_info = manager
            .download_model(&model_id, None, false)
            .await
            .expect("Failed to download model");
        assert_eq!(model_info.id, model_id);

        // Verify model is cached
        let cached_models = manager
            .list_cached_models()
            .await
            .expect("Failed to list cached models");
        assert!(
            cached_models.iter().any(|m| m.id == model_id),
            "Downloaded model not found in cache"
        );

        // Load the model
        let model_info = manager
            .load_model(&model_id, Some(300))
            .await
            .expect("Failed to load model");
        assert_eq!(model_info.id, model_id);

        // Verify model is loaded
        let loaded_models = manager
            .list_loaded_models()
            .await
            .expect("Failed to list loaded models");
        assert!(loaded_models.iter().any(|m| m.id == model_id));

        // Get the endpoint before making the request
        let endpoint = manager.endpoint().expect("Failed to get endpoint");

        // Use the OpenAI compatible API to interact with the model
        let client = reqwest::Client::new();
        let response = client
            .post(&format!("{}/chat/completions", endpoint))
            .json(&serde_json::json!({
                "model": model_info.id,
                "messages": [{"role": "user", "content": "What is 2+2?"}],
            }))
            .send()
            .await
            .expect("Failed to send request");

        // Parse and display the response
        let result = response
            .json::<serde_json::Value>()
            .await
            .expect("Failed to parse response");
        if let Some(content) = result["choices"][0]["message"]["content"].as_str() {
            println!("\nResponse:\n{}", content);
        } else {
            println!("\nError: Failed to extract response content from API result");
            println!("Full API response: {}", result);
        }

        // Unload the model
        manager
            .unload_model(&model_id, true)
            .await
            .expect("Failed to unload model");

        // Verify model is unloaded
        let loaded_models = manager
            .list_loaded_models()
            .await
            .expect("Failed to list loaded models");
        assert!(!loaded_models.iter().any(|m| m.id == model_id));
    }
}