dwctl 8.71.0

The Doubleword Control Layer - A self-hostable observability and analytics platform for LLM applications
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
//! Error types and HTTP response conversion.
//!
//! This module defines the application's error hierarchy and implements conversion
//! to HTTP responses with appropriate status codes and JSON payloads.
//!
//! # Error Hierarchy
//!
//! The main [`Error`] enum covers all application error cases:
//!
//! - **Authentication Errors**: `Unauthenticated` (401)
//! - **Authorization Errors**: `InsufficientPermissions` (403)
//! - **Validation Errors**: `BadRequest` (400)
//! - **Not Found Errors**: `NotFound` (404)
//! - **Conflict Errors**: `Conflict` (409) for unique constraint violations
//! - **Database Errors**: Wraps [`DbError`] with appropriate status codes
//! - **Internal Errors**: Generic server errors (500)
//!
//! # HTTP Response Conversion
//!
//! All errors implement [`IntoResponse`] for automatic conversion to HTTP responses
//! with JSON bodies:
//!
//! ```json
//! {
//!   "error": "Not Found",
//!   "message": "User with ID abc123 not found"
//! }
//! ```
//!
//! # Usage in Handlers
//!
//! Handlers can return `Result<T, Error>` and errors will automatically convert
//! to appropriate HTTP responses:
//!
//! ```ignore
//! use dwctl::errors::Error;
//!
//! async fn handler() -> Result<String, Error> {
//!     Err(Error::BadRequest {
//!         message: "Invalid input".to_string()
//!     })
//! }
//! ```
//!
//! # Error Construction Helpers
//!
//! The module provides convenience methods for common error types:
//!
//! ```ignore
//! // Not found error
//! return Err(Error::NotFound {
//!     resource: "User".to_string(),
//!     id: user_id.to_string(),
//! });
//!
//! // Permission error
//! return Err(Error::InsufficientPermissions {
//!     required: Permission::Admin,
//!     action: Operation::Delete,
//!     resource: "deployment".to_string(),
//! });
//! ```

use crate::db::errors::DbError;
use crate::types::{Operation, Permission};
use axum::{
    http::StatusCode,
    response::{IntoResponse, Response},
};
use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
use thiserror::Error as ThisError;

/// Retry-After header value (in seconds) for 503 Service Unavailable responses
/// when the database connection pool is exhausted.
const POOL_EXHAUSTED_RETRY_AFTER_SECS: &str = "30";

#[derive(ThisError, Debug)]
pub enum Error {
    /// Authentication required but not provided
    #[error("Not authenticated")]
    Unauthenticated { message: Option<String> },

    /// User lacks required permissions for the operation
    #[error("Insufficient permissions to {action:?} {resource}")]
    InsufficientPermissions {
        required: Permission,
        action: Operation,
        resource: String,
    },

    /// Invalid request data or business rule violation
    #[error("{message}")]
    BadRequest { message: String },

    /// The request was well-formed but references something we cannot
    /// process — e.g. an image URL whose origin returned a 4xx (forbidden,
    /// gated, missing). The caller's input is at fault, but the request
    /// itself is not malformed, so this is 422 rather than 400.
    #[error("{message}")]
    UnprocessableEntity { message: String },

    /// Requested resource not found
    #[error("{resource} with ID {id} not found")]
    NotFound { resource: String, id: String },

    /// Resource existed but is permanently gone (e.g. a zero-data-retention body
    /// whose key was deleted or expired). Maps to 410.
    #[error("{message}")]
    Gone { message: String },

    /// Generic internal service error
    #[error("Failed to {operation}")]
    Internal { operation: String },

    /// Database operation error
    #[error(transparent)]
    Database(#[from] DbError),

    /// Unexpected error with full context chain
    #[error(transparent)]
    Other(#[from] anyhow::Error),

    /// Conflict error, e.g., for unique constraint violations
    #[error("Conflict: {message}")]
    Conflict {
        message: String,
        conflicts: Option<Vec<AliasConflict>>,
    },

    /// Payload exceeds maximum allowed size
    #[error("Payload too large: {message}")]
    PayloadTooLarge { message: String },

    /// Insufficient credits to perform the requested operation
    #[error("Insufficient credits: {message}")]
    InsufficientCredits { current_balance: Decimal, message: String },

    /// User does not have access to the requested model
    #[error("Model access denied: {message}")]
    ModelAccessDenied { model_name: String, message: String },

    /// API key's purpose (modality) is denied for the requested model by a routing rule
    #[error("Modality access denied: {message}")]
    ModalityAccessDenied {
        model_name: String,
        purpose: String,
        message: String,
    },

    /// Too many concurrent requests - rate limiting
    #[error("Too many requests: {message}")]
    TooManyRequests { message: String },

    /// A transient dependency failure that the client should retry
    /// (e.g. an upstream fetch timed out after retries, or a backing
    /// store was briefly unreachable). Produces 503 so retry-aware
    /// clients back off and retry rather than treating it as a
    /// permanent 4xx/5xx.
    #[error("Service temporarily unavailable: {message}")]
    ServiceUnavailable { message: String },
}

#[derive(Debug, Serialize, Deserialize)]
pub struct AliasConflict {
    pub model_name: String,
    pub attempted_alias: String,
}

impl Error {
    pub fn status_code(&self) -> StatusCode {
        match self {
            Error::Unauthenticated { .. } => StatusCode::UNAUTHORIZED,
            Error::InsufficientPermissions { .. } => StatusCode::FORBIDDEN,
            Error::BadRequest { .. } => StatusCode::BAD_REQUEST,
            Error::UnprocessableEntity { .. } => StatusCode::UNPROCESSABLE_ENTITY,
            Error::NotFound { .. } => StatusCode::NOT_FOUND,
            Error::Gone { .. } => StatusCode::GONE,
            Error::Internal { .. } => StatusCode::INTERNAL_SERVER_ERROR,
            Error::Database(db_err) => match db_err {
                DbError::NotFound => StatusCode::NOT_FOUND,
                DbError::UniqueViolation { .. } => StatusCode::CONFLICT,
                DbError::ForeignKeyViolation { .. } => StatusCode::BAD_REQUEST,
                DbError::CheckViolation { .. } => StatusCode::BAD_REQUEST,
                DbError::ProtectedEntity { .. } => StatusCode::FORBIDDEN,
                DbError::InvalidModelField { .. } => StatusCode::BAD_REQUEST,
                DbError::PoolExhausted => StatusCode::SERVICE_UNAVAILABLE,
                DbError::Other(_) => StatusCode::INTERNAL_SERVER_ERROR,
            },
            Error::Other(_) => StatusCode::INTERNAL_SERVER_ERROR,
            Error::Conflict { .. } => StatusCode::CONFLICT,
            Error::PayloadTooLarge { .. } => StatusCode::PAYLOAD_TOO_LARGE,
            Error::InsufficientCredits { .. } => StatusCode::PAYMENT_REQUIRED,
            Error::ModelAccessDenied { .. } => StatusCode::FORBIDDEN,
            Error::ModalityAccessDenied { .. } => StatusCode::FORBIDDEN,
            Error::TooManyRequests { .. } => StatusCode::TOO_MANY_REQUESTS,
            Error::ServiceUnavailable { .. } => StatusCode::SERVICE_UNAVAILABLE,
        }
    }

    /// Returns a user-safe error message, without leaking internal implementation details
    pub fn user_message(&self) -> String {
        match self {
            Error::Unauthenticated { message } => message.clone().unwrap_or_else(|| "Authentication required".to_string()),
            Error::InsufficientPermissions { action, resource, .. } => {
                format!("Insufficient permissions to {action} {resource}")
            }
            Error::BadRequest { message } => message.clone(),
            Error::UnprocessableEntity { message } => message.clone(),
            Error::PayloadTooLarge { message } => message.clone(),
            Error::NotFound { resource, id } => {
                format!("{resource} with ID {id} not found")
            }
            Error::Gone { message } => message.clone(),
            Error::Internal { .. } => "Internal server error".to_string(),
            Error::Database(db_err) => match db_err {
                DbError::NotFound => "Resource not found".to_string(),
                DbError::UniqueViolation { constraint, table, .. } => {
                    // Provide user-friendly messages for common unique constraint violations
                    match (table.as_deref(), constraint.as_deref()) {
                        (Some("users"), Some(c)) if c.contains("email") => "An account with this email address already exists".to_string(),
                        (Some("users"), Some(c)) if c.contains("username") => "This username is already taken".to_string(),
                        (Some("deployed_models"), Some("deployed_models_alias_unique")) => {
                            "The specified alias is already in use. Please choose a different alias.".to_string()
                        }
                        (Some("user_organizations"), Some(c)) if c.contains("invite_email") => {
                            "A pending invite already exists for this email address".to_string()
                        }
                        _ => "Resource already exists".to_string(),
                    }
                }
                DbError::ForeignKeyViolation { .. } => "Invalid reference to related resource".to_string(),
                DbError::CheckViolation { .. } => "Invalid data provided".to_string(),
                DbError::ProtectedEntity {
                    operation,
                    entity_type,
                    reason,
                    ..
                } => {
                    format!("Cannot {operation:?} {entity_type}: {reason}")
                }
                DbError::InvalidModelField { field } => format!("Field '{field}' must not be empty or whitespace"),
                DbError::PoolExhausted => "Service temporarily overloaded, please retry".to_string(),
                DbError::Other(_) => "Database error occurred".to_string(),
            },
            Error::Other(_) => "Internal server error".to_string(),
            Error::Conflict { message, conflicts } => {
                if let Some(conflicts) = conflicts {
                    let aliases: Vec<String> = conflicts.iter().map(|c| c.attempted_alias.to_string()).collect();
                    format!("{message}: {}", aliases.join(", "))
                } else {
                    message.clone()
                }
            }
            Error::InsufficientCredits { message, .. } => message.clone(),
            Error::ModelAccessDenied { message, .. } => message.clone(),
            Error::ModalityAccessDenied { message, .. } => message.clone(),
            Error::TooManyRequests { message } => message.clone(),
            Error::ServiceUnavailable { message } => message.clone(),
        }
    }
}

impl IntoResponse for Error {
    fn into_response(self) -> Response {
        // Log full error details for debugging - different log levels based on severity
        match &self {
            Error::Database(DbError::Other(_)) | Error::Internal { .. } | Error::Other(_) => {
                tracing::error!("Internal service error: {:#}", self);
            }
            Error::Database(DbError::PoolExhausted) => {
                tracing::warn!("Database connection pool exhausted - service overloaded");
            }
            Error::Database(_) => {
                tracing::warn!("Database constraint error: {}", self);
            }
            Error::Unauthenticated { .. } | Error::InsufficientPermissions { .. } => {
                tracing::info!("Authorization error: {}", self);
            }
            Error::BadRequest { .. }
            | Error::UnprocessableEntity { .. }
            | Error::NotFound { .. }
            | Error::Gone { .. }
            | Error::PayloadTooLarge { .. } => {
                tracing::debug!("Client error: {}", self);
            }
            Error::Conflict { .. } => {
                tracing::warn!("Conflict error: {}", self);
            }
            Error::InsufficientCredits { .. } => {
                tracing::info!("Insufficient credits error: {}", self);
            }
            Error::ModelAccessDenied { .. } => {
                tracing::info!("Model access denied error: {}", self);
            }
            Error::ModalityAccessDenied { .. } => {
                tracing::info!("Modality access denied error: {}", self);
            }
            Error::TooManyRequests { .. } => {
                tracing::info!("Rate limit exceeded: {}", self);
            }
            Error::ServiceUnavailable { .. } => {
                tracing::warn!("Service temporarily unavailable: {}", self);
            }
        }

        let status = self.status_code();

        // Handle structured JSON responses for specific error types
        match &self {
            Error::Conflict { message, conflicts } => {
                use serde_json::json;
                let body = if let Some(conflicts) = conflicts {
                    json!({
                        "message": message,
                        "conflicts": conflicts
                    })
                } else {
                    json!({ "message": message })
                };

                (status, axum::response::Json(body)).into_response()
            }
            // Handle pool exhaustion with Retry-After header
            Error::Database(DbError::PoolExhausted) => {
                use axum::http::header::RETRY_AFTER;
                use serde_json::json;
                let body = json!({
                    "error": "service_unavailable",
                    "message": self.user_message(),
                    "retry_after_seconds": 30
                });
                (status, [(RETRY_AFTER, POOL_EXHAUSTED_RETRY_AFTER_SECS)], axum::response::Json(body)).into_response()
            }
            // Handle database unique violations with minimal structured JSON
            Error::Database(DbError::UniqueViolation { constraint, table, .. }) => {
                use serde_json::json;

                // Determine the resource and message only
                let (message, resource) = match (table.as_deref(), constraint.as_deref()) {
                    (Some("users"), Some(c)) if c.contains("email") => {
                        ("An account with this email address already exists".to_string(), "user")
                    }
                    (Some("users"), Some(c)) if c.contains("username") => ("This username is already taken".to_string(), "user"),
                    (Some("deployed_models"), Some("deployed_models_alias_unique")) => (
                        "The specified alias is already in use. Please choose a different alias.".to_string(),
                        "deployment",
                    ),
                    (Some("inference_endpoints"), Some(c)) if c.contains("name") => {
                        ("An endpoint with this name already exists".to_string(), "endpoint")
                    }
                    (Some("inference_endpoints"), Some(c)) if c.contains("url") => {
                        ("An endpoint with this URL already exists".to_string(), "endpoint")
                    }
                    _ => ("Resource already exists".to_string(), "unknown"),
                };

                let body = json!({
                    "message": message,
                    "resource": resource
                });

                (status, axum::response::Json(body)).into_response()
            }
            Error::TooManyRequests { message } => {
                use axum::http::header::RETRY_AFTER;
                use serde_json::json;

                // Suggest retry after 60 seconds for capacity-based rejections
                let retry_after_secs = "60";

                let body = json!({
                    "error": "too_many_requests",
                    "message": message,
                    "retry_after_seconds": 30
                });

                (status, [(RETRY_AFTER, retry_after_secs)], axum::response::Json(body)).into_response()
            }
            Error::ServiceUnavailable { message } => {
                use axum::http::header::RETRY_AFTER;
                use serde_json::json;
                let body = json!({
                    "error": "service_unavailable",
                    "message": message,
                    "retry_after_seconds": 30
                });
                (status, [(RETRY_AFTER, "30")], axum::response::Json(body)).into_response()
            }
            _ => {
                // For all other errors, return simple text message (unchanged)
                let user_message = self.user_message();
                (status, user_message).into_response()
            }
        }
    }
}

/// Convert from String errors (e.g., from external functions)
impl From<String> for Error {
    fn from(msg: String) -> Self {
        Error::Internal { operation: msg }
    }
}

/// Type alias for service operation results
pub type Result<T> = std::result::Result<T, Error>;