1use super::*;
2
3#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
4#[serde(rename_all = "camelCase")]
5pub enum ResourceUsageStatus {
6 Fresh,
7 Unavailable,
8}
9
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
11#[serde(rename_all = "camelCase")]
12pub enum ResourcePressureDimension {
13 FilesystemBytes,
14 ProviderQuota,
15 Inodes,
16}
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "camelCase")]
20pub enum ResourcePressureSeverity {
21 Normal,
22 Warning,
23 Critical,
24}
25
26#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct ResourceUsageValue {
29 pub total: u64,
30 pub used: u64,
31 pub available: u64,
32 pub usage_percent: f64,
33}
34
35#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
36#[serde(rename_all = "camelCase")]
37pub struct ResourcePressure {
38 pub dimension: ResourcePressureDimension,
39 pub usage_percent: f64,
40 pub severity: ResourcePressureSeverity,
41 #[serde(default, skip_serializing_if = "Option::is_none")]
42 pub remaining_bytes: Option<u64>,
43}
44
45#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
46#[serde(rename_all = "camelCase")]
47pub struct WorkspaceStorageUsage {
48 #[serde(default, skip_serializing_if = "Option::is_none")]
49 pub filesystem: Option<ResourceUsageValue>,
50 #[serde(default, skip_serializing_if = "Option::is_none")]
51 pub provider_quota: Option<ResourceUsageValue>,
52 #[serde(default, skip_serializing_if = "Option::is_none")]
53 pub inodes: Option<ResourceUsageValue>,
54 #[serde(default, skip_serializing_if = "Option::is_none")]
55 pub pressure: Option<ResourcePressure>,
56}
57
58#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
59#[serde(rename_all = "camelCase")]
60pub struct WorkspaceResourceUsage {
61 pub schema_version: String,
62 pub status: ResourceUsageStatus,
63 pub collected_at_ms: u64,
64 pub workspace: String,
65 pub root: PathBuf,
66 #[serde(default, skip_serializing_if = "Option::is_none")]
67 pub backend_id: Option<String>,
68 pub storage: WorkspaceStorageUsage,
69 #[serde(default, skip_serializing_if = "Vec::is_empty")]
70 pub warnings: Vec<String>,
71}
72
73impl WorkspaceResourceUsage {
74 pub fn unavailable(
75 workspace: String,
76 root: PathBuf,
77 backend_id: Option<String>,
78 warning: impl Into<String>,
79 ) -> Self {
80 Self {
81 schema_version: "v1".to_string(),
82 status: ResourceUsageStatus::Unavailable,
83 collected_at_ms: resource_usage_now_ms(),
84 workspace,
85 root,
86 backend_id,
87 storage: WorkspaceStorageUsage {
88 filesystem: None,
89 provider_quota: None,
90 inodes: None,
91 pressure: None,
92 },
93 warnings: vec![warning.into()],
94 }
95 }
96}
97
98pub const WORKSPACE_RESOURCE_EXHAUSTED_CODE: &str = "RESOURCE_EXHAUSTED";
99
100#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
101#[serde(rename_all = "camelCase")]
102pub struct WorkspaceErrorDetails {
103 pub schema_version: String,
104 pub code: String,
105 pub message: String,
106 #[serde(default, skip_serializing_if = "Option::is_none")]
107 pub cause: Option<String>,
108 pub scope: String,
109 pub operation: String,
110 #[serde(default, skip_serializing_if = "Option::is_none")]
111 pub resource: Option<ResourcePressureDimension>,
112 pub retryable: bool,
113 #[serde(default, skip_serializing_if = "Option::is_none")]
114 pub requested_bytes: Option<u64>,
115 #[serde(default, skip_serializing_if = "Option::is_none")]
116 pub available_bytes: Option<u64>,
117 #[serde(default, skip_serializing_if = "Option::is_none")]
118 pub reserve_bytes: Option<u64>,
119 #[serde(default, skip_serializing_if = "Option::is_none")]
120 pub requested_inodes: Option<u64>,
121 #[serde(default, skip_serializing_if = "Option::is_none")]
122 pub available_inodes: Option<u64>,
123 #[serde(default, skip_serializing_if = "Option::is_none")]
124 pub reserve_inodes: Option<u64>,
125 #[serde(default, skip_serializing_if = "Option::is_none")]
126 pub workspace: Option<String>,
127 #[serde(default, skip_serializing_if = "Option::is_none")]
128 pub root: Option<PathBuf>,
129 #[serde(default, skip_serializing_if = "Option::is_none")]
130 pub backend_id: Option<String>,
131 #[serde(default, skip_serializing_if = "Option::is_none")]
132 pub usage: Option<WorkspaceResourceUsage>,
133}
134
135impl WorkspaceErrorDetails {
136 pub fn resource_exhausted(operation: impl Into<String>, message: impl Into<String>) -> Self {
137 Self {
138 schema_version: "v1".to_string(),
139 code: WORKSPACE_RESOURCE_EXHAUSTED_CODE.to_string(),
140 message: message.into(),
141 cause: None,
142 scope: "sandbox".to_string(),
143 operation: operation.into(),
144 resource: None,
145 retryable: false,
146 requested_bytes: None,
147 available_bytes: None,
148 reserve_bytes: None,
149 requested_inodes: None,
150 available_inodes: None,
151 reserve_inodes: None,
152 workspace: None,
153 root: None,
154 backend_id: None,
155 usage: None,
156 }
157 }
158}
159
160#[derive(Debug, Clone)]
161pub struct WorkspaceOperationError {
162 details: WorkspaceErrorDetails,
163}
164
165impl WorkspaceOperationError {
166 pub fn new(details: WorkspaceErrorDetails) -> Self {
167 Self { details }
168 }
169
170 pub fn details(&self) -> &WorkspaceErrorDetails {
171 &self.details
172 }
173}
174
175impl fmt::Display for WorkspaceOperationError {
176 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
177 formatter.write_str(&self.details.message)
178 }
179}
180
181impl std::error::Error for WorkspaceOperationError {}
182
183pub fn workspace_error_details(error: &anyhow::Error) -> Option<&WorkspaceErrorDetails> {
184 error
185 .downcast_ref::<WorkspaceOperationError>()
186 .map(WorkspaceOperationError::details)
187}
188
189pub fn is_workspace_resource_exhausted_message(message: &str) -> bool {
190 let normalized = message.to_ascii_lowercase();
191 normalized.contains("no space left on device")
192 || normalized.contains("os error 28")
193 || normalized.contains("enospc")
194 || normalized.contains("insufficient storage")
195}
196
197pub fn normalize_workspace_operation_error(
198 error: anyhow::Error,
199 operation: &str,
200 workspace: &dyn Workspace,
201) -> anyhow::Error {
202 let cause = format!("{error:#}");
203 if workspace_error_details(&error).is_some() || !is_workspace_resource_exhausted_message(&cause)
204 {
205 return error;
206 }
207 let mut details = WorkspaceErrorDetails::resource_exhausted(
208 operation,
209 format!("{operation} failed because the sandbox has no writable storage capacity"),
210 );
211 details.cause = Some(cause);
212 details.workspace = Some(workspace.description());
213 details.root = Some(workspace.root());
214 details.backend_id = resolved_backend_id_value(workspace.resolved_backend_id());
215 WorkspaceOperationError::new(details).into()
216}
217
218fn resource_usage_now_ms() -> u64 {
219 SystemTime::now()
220 .duration_since(UNIX_EPOCH)
221 .unwrap_or_default()
222 .as_millis()
223 .try_into()
224 .unwrap_or(u64::MAX)
225}
226
227fn resolved_backend_id_value(id: Option<ResolvedBackendId>) -> Option<String> {
228 id.map(|value| value.0)
229}
230
231#[doc(hidden)]
232pub fn resource_pressure(
233 filesystem: &ResourceUsageValue,
234 provider_quota: Option<&ResourceUsageValue>,
235 inodes: &ResourceUsageValue,
236) -> ResourcePressure {
237 let mut pressure = (
238 ResourcePressureDimension::FilesystemBytes,
239 filesystem.usage_percent,
240 Some(filesystem.available),
241 );
242 if let Some(quota) = provider_quota
243 && quota.usage_percent > pressure.1
244 {
245 pressure = (
246 ResourcePressureDimension::ProviderQuota,
247 quota.usage_percent,
248 Some(quota.available),
249 );
250 }
251 if inodes.usage_percent > pressure.1 {
252 pressure = (
253 ResourcePressureDimension::Inodes,
254 inodes.usage_percent,
255 None,
256 );
257 }
258 let (dimension, usage_percent, remaining_bytes) = pressure;
259 let severity = if usage_percent >= 90.0 {
260 ResourcePressureSeverity::Critical
261 } else if usage_percent >= 80.0 {
262 ResourcePressureSeverity::Warning
263 } else {
264 ResourcePressureSeverity::Normal
265 };
266 ResourcePressure {
267 dimension,
268 usage_percent,
269 severity,
270 remaining_bytes,
271 }
272}
273
274#[doc(hidden)]
275pub fn attach_provider_quota(
276 usage: &mut WorkspaceResourceUsage,
277 provider_quota: ResourceUsageValue,
278) {
279 usage
280 .warnings
281 .retain(|warning| warning != "providerQuotaUnavailable");
282 usage.storage.provider_quota = Some(provider_quota);
283 if let (Some(filesystem), Some(provider_quota), Some(inodes)) = (
284 usage.storage.filesystem.as_ref(),
285 usage.storage.provider_quota.as_ref(),
286 usage.storage.inodes.as_ref(),
287 ) {
288 usage.storage.pressure = Some(resource_pressure(filesystem, Some(provider_quota), inodes));
289 }
290}