use super::*;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum ResourceUsageStatus {
Fresh,
Unavailable,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum ResourcePressureDimension {
FilesystemBytes,
ProviderQuota,
Inodes,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum ResourcePressureSeverity {
Normal,
Warning,
Critical,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ResourceUsageValue {
pub total: u64,
pub used: u64,
pub available: u64,
pub usage_percent: f64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ResourcePressure {
pub dimension: ResourcePressureDimension,
pub usage_percent: f64,
pub severity: ResourcePressureSeverity,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub remaining_bytes: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct WorkspaceStorageUsage {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub filesystem: Option<ResourceUsageValue>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_quota: Option<ResourceUsageValue>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub inodes: Option<ResourceUsageValue>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pressure: Option<ResourcePressure>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct WorkspaceResourceUsage {
pub schema_version: String,
pub status: ResourceUsageStatus,
pub collected_at_ms: u64,
pub workspace: String,
pub root: PathBuf,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub backend_id: Option<String>,
pub storage: WorkspaceStorageUsage,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub warnings: Vec<String>,
}
impl WorkspaceResourceUsage {
pub fn unavailable(
workspace: String,
root: PathBuf,
backend_id: Option<String>,
warning: impl Into<String>,
) -> Self {
Self {
schema_version: "v1".to_string(),
status: ResourceUsageStatus::Unavailable,
collected_at_ms: resource_usage_now_ms(),
workspace,
root,
backend_id,
storage: WorkspaceStorageUsage {
filesystem: None,
provider_quota: None,
inodes: None,
pressure: None,
},
warnings: vec![warning.into()],
}
}
}
pub const WORKSPACE_RESOURCE_EXHAUSTED_CODE: &str = "RESOURCE_EXHAUSTED";
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct WorkspaceErrorDetails {
pub schema_version: String,
pub code: String,
pub message: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cause: Option<String>,
pub scope: String,
pub operation: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub resource: Option<ResourcePressureDimension>,
pub retryable: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub requested_bytes: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub available_bytes: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reserve_bytes: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub requested_inodes: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub available_inodes: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reserve_inodes: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub workspace: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub root: Option<PathBuf>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub backend_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage: Option<WorkspaceResourceUsage>,
}
impl WorkspaceErrorDetails {
pub fn resource_exhausted(operation: impl Into<String>, message: impl Into<String>) -> Self {
Self {
schema_version: "v1".to_string(),
code: WORKSPACE_RESOURCE_EXHAUSTED_CODE.to_string(),
message: message.into(),
cause: None,
scope: "sandbox".to_string(),
operation: operation.into(),
resource: None,
retryable: false,
requested_bytes: None,
available_bytes: None,
reserve_bytes: None,
requested_inodes: None,
available_inodes: None,
reserve_inodes: None,
workspace: None,
root: None,
backend_id: None,
usage: None,
}
}
}
#[derive(Debug, Clone)]
pub struct WorkspaceOperationError {
details: WorkspaceErrorDetails,
}
impl WorkspaceOperationError {
pub fn new(details: WorkspaceErrorDetails) -> Self {
Self { details }
}
pub fn details(&self) -> &WorkspaceErrorDetails {
&self.details
}
}
impl fmt::Display for WorkspaceOperationError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.details.message)
}
}
impl std::error::Error for WorkspaceOperationError {}
pub fn workspace_error_details(error: &anyhow::Error) -> Option<&WorkspaceErrorDetails> {
error
.downcast_ref::<WorkspaceOperationError>()
.map(WorkspaceOperationError::details)
}
pub fn is_workspace_resource_exhausted_message(message: &str) -> bool {
let normalized = message.to_ascii_lowercase();
normalized.contains("no space left on device")
|| normalized.contains("os error 28")
|| normalized.contains("enospc")
|| normalized.contains("insufficient storage")
}
pub fn normalize_workspace_operation_error(
error: anyhow::Error,
operation: &str,
workspace: &dyn Workspace,
) -> anyhow::Error {
let cause = format!("{error:#}");
if workspace_error_details(&error).is_some() || !is_workspace_resource_exhausted_message(&cause)
{
return error;
}
let mut details = WorkspaceErrorDetails::resource_exhausted(
operation,
format!("{operation} failed because the sandbox has no writable storage capacity"),
);
details.cause = Some(cause);
details.workspace = Some(workspace.description());
details.root = Some(workspace.root());
details.backend_id = resolved_backend_id_value(workspace.resolved_backend_id());
WorkspaceOperationError::new(details).into()
}
fn resource_usage_now_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
.try_into()
.unwrap_or(u64::MAX)
}
fn resolved_backend_id_value(id: Option<ResolvedBackendId>) -> Option<String> {
id.map(|value| value.0)
}
#[doc(hidden)]
pub fn resource_pressure(
filesystem: &ResourceUsageValue,
provider_quota: Option<&ResourceUsageValue>,
inodes: &ResourceUsageValue,
) -> ResourcePressure {
let mut pressure = (
ResourcePressureDimension::FilesystemBytes,
filesystem.usage_percent,
Some(filesystem.available),
);
if let Some(quota) = provider_quota
&& quota.usage_percent > pressure.1
{
pressure = (
ResourcePressureDimension::ProviderQuota,
quota.usage_percent,
Some(quota.available),
);
}
if inodes.usage_percent > pressure.1 {
pressure = (
ResourcePressureDimension::Inodes,
inodes.usage_percent,
None,
);
}
let (dimension, usage_percent, remaining_bytes) = pressure;
let severity = if usage_percent >= 90.0 {
ResourcePressureSeverity::Critical
} else if usage_percent >= 80.0 {
ResourcePressureSeverity::Warning
} else {
ResourcePressureSeverity::Normal
};
ResourcePressure {
dimension,
usage_percent,
severity,
remaining_bytes,
}
}
#[doc(hidden)]
pub fn attach_provider_quota(
usage: &mut WorkspaceResourceUsage,
provider_quota: ResourceUsageValue,
) {
usage
.warnings
.retain(|warning| warning != "providerQuotaUnavailable");
usage.storage.provider_quota = Some(provider_quota);
if let (Some(filesystem), Some(provider_quota), Some(inodes)) = (
usage.storage.filesystem.as_ref(),
usage.storage.provider_quota.as_ref(),
usage.storage.inodes.as_ref(),
) {
usage.storage.pressure = Some(resource_pressure(filesystem, Some(provider_quota), inodes));
}
}