foundry-local-sdk 1.2.0-rc1

Local AI model inference powered by the Foundry Local Core engine
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

use serde::{Deserialize, Serialize};

/// Hardware device type for model execution.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub enum DeviceType {
    Invalid,
    #[default]
    CPU,
    GPU,
    NPU,
}

/// Prompt template describing how messages are formatted for the model.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PromptTemplate {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub system: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub user: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub assistant: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub prompt: Option<String>,
}

/// Runtime information for a model (device type and execution provider).
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Runtime {
    pub device_type: DeviceType,
    pub execution_provider: String,
}

/// A single parameter key-value pair used in model settings.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Parameter {
    pub name: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub value: Option<String>,
}

/// Model-level settings containing a list of parameters.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ModelSettings {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub parameters: Option<Vec<Parameter>>,
}

/// Full metadata for a model variant as returned by the catalog.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ModelInfo {
    pub id: String,
    pub name: String,
    pub version: u64,
    pub alias: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub display_name: Option<String>,
    pub provider_type: String,
    pub uri: String,
    pub model_type: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub prompt_template: Option<PromptTemplate>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub publisher: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub model_settings: Option<ModelSettings>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub license: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub license_description: Option<String>,
    pub cached: bool,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub task: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub runtime: Option<Runtime>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub file_size_mb: Option<u64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub supports_tool_calling: Option<bool>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub max_output_tokens: Option<u64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub min_fl_version: Option<String>,
    #[serde(default)]
    pub created_at_unix: u64,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub context_length: Option<u64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub input_modalities: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub output_modalities: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub capabilities: Option<String>,
}

/// Desired response format for chat completions.
///
/// Extends the standard OpenAI formats with the Foundry-specific
/// `LarkGrammar` variant.
#[derive(Debug, Clone)]
pub enum ChatResponseFormat {
    /// Plain text output (default).
    Text,
    /// JSON output (unstructured).
    JsonObject,
    /// JSON output constrained by a schema string.
    JsonSchema(String),
    /// Output constrained by a Lark grammar (Foundry extension).
    LarkGrammar(String),
}

/// Tool choice configuration for chat completions.
#[derive(Debug, Clone)]
pub enum ChatToolChoice {
    /// Model will not call any tool.
    None,
    /// Model decides whether to call a tool.
    Auto,
    /// Model must call at least one tool.
    Required,
    /// Model must call the named function.
    Function(String),
}

/// Information about an available execution provider.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "PascalCase")]
pub struct EpInfo {
    /// The name of the execution provider.
    pub name: String,
    /// Whether this EP is currently registered and ready for use.
    pub is_registered: bool,
}

/// Result of a download-and-register execution-provider operation.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "PascalCase")]
pub struct EpDownloadResult {
    /// Whether all requested EPs were successfully registered.
    pub success: bool,
    /// Human-readable status message.
    pub status: String,
    /// Names of EPs that were successfully registered.
    pub registered_eps: Vec<String>,
    /// Names of EPs that failed to register.
    pub failed_eps: Vec<String>,
}