tensorlake 0.5.157

A Rust SDK for interacting with Tensorlake Cloud APIs.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
//! # SDK Error Types
//!
//! This module provides comprehensive error handling for the Tensorlake Cloud SDK.
//! It includes a general `SdkError` type that encompasses all possible error scenarios
//! across the different clients, including authentication and authorization errors.

use thiserror::Error;

use crate::{
    applications::error::ApplicationsError, images::error::ImagesError,
    secrets::error::SecretsError,
};

/// The main error type for the Tensorlake Cloud SDK.
///
/// This enum encompasses all possible errors that can occur when using the SDK,
/// including client-specific errors, authentication issues, and general HTTP errors.
#[derive(Debug, Error)]
pub enum SdkError {
    /// A safe read has already used its retry policy. Binding-level retry
    /// wrappers must not start another sequence. The original error is kept
    /// for language bindings and callers inspecting the failure.
    #[error(transparent)]
    ReadRetryExhausted(Box<SdkError>),
    /// A resize failed or could not be observed to completion.
    #[error(transparent)]
    SandboxResize(#[from] Box<crate::sandboxes::resize::SandboxResizeError>),
    /// Errors specific to the Applications client
    #[error(transparent)]
    Applications(#[from] ApplicationsError),

    /// Authentication error (HTTP 401)
    #[error("Authentication failed: {0}")]
    Authentication(String),

    /// Authorization error (HTTP 403)
    #[error("Authorization failed: {0}")]
    Authorization(String),

    /// General HTTP errors
    #[error(transparent)]
    Http(#[from] reqwest::Error),

    /// Reqwest middleware errors
    #[error(transparent)]
    Middleware(#[from] reqwest_middleware::Error),

    /// Errors specific to the Images client
    #[error(transparent)]
    Images(#[from] ImagesError),

    /// Invalid header value during client initialization
    #[error("Invalid header value: {0}")]
    InvalidHeaderValue(String),

    /// Client configuration error
    #[error("Client error: {0}")]
    ClientError(String),

    /// General IO errors
    #[error(transparent)]
    Io(#[from] std::io::Error),

    /// JSON serialization/deserialization errors
    #[error(transparent)]
    Json(#[from] serde_json::Error),

    /// JSON serialization/deserialization errors
    #[error(transparent)]
    JsonWithError(#[from] serde_path_to_error::Error<serde_json::Error>),

    /// Errors specific to the Secrets client
    #[error(transparent)]
    Secrets(#[from] SecretsError),

    /// Server returned an error status
    #[error("Server error: {status} - {message}")]
    ServerError {
        status: reqwest::StatusCode,
        message: String,
    },

    /// EventSource (SSE) stream error
    #[error("EventSource error: {0}")]
    EventSourceError(String),

    /// The connection failed while an EventSource (SSE) response body was
    /// streaming. reqwest reports every such failure as "error decoding
    /// response body"; the cause is in the error's source chain.
    #[error("EventSource error: {0}")]
    EventStreamTransport(#[source] reqwest::Error),

    /// A streamed response did not start, or stopped sending data, within the
    /// client's request timeout.
    #[error("timed out after {timeout:?} waiting for {waiting_for}")]
    StreamTimeout {
        waiting_for: &'static str,
        timeout: std::time::Duration,
    },
}

/// Error codes the Sandbox Proxy returns when it gave up on a request *before*
/// any dataplane executed it, so replaying the request cannot duplicate a side
/// effect:
///
/// - `SANDBOX_UPSTREAM_ERROR` — the Indexify lookup did not answer.
/// - `SANDBOX_NOT_READY` — the sandbox has no route yet.
/// - `SANDBOX_UNREACHABLE` — the dataplane could not be connected to, or a
///   dataplane answered that it does not host the sandbox (its own
///   `SANDBOX_NOT_FOUND` / `SANDBOX_NOT_RUNNING`, sent instead of executing
///   the request) and the route could not be refreshed in time.
/// - `AUTH_SERVICE_UNAVAILABLE` — the Platform auth check failed; nothing was
///   forwarded.
///
/// These are what an Indexify server rollout looks like from the client. The
/// proxy's error body is `{"error": "...", "code": "..."}`. Proxies older than
/// compute-engine-internal #2083 report the lookup gap as a plain
/// `SANDBOX_NOT_FOUND` 404, which is not replayed.
pub const SANDBOX_PROXY_UNDELIVERED_CODES: &[&str] = &[
    "SANDBOX_UPSTREAM_ERROR",
    "SANDBOX_NOT_READY",
    "SANDBOX_UNREACHABLE",
    "AUTH_SERVICE_UNAVAILABLE",
];

/// Why a request failed below the HTTP status layer.
///
/// The distinction is load-bearing for retries: a [`TransportFailure::Connect`]
/// error means the request was never written to the wire, so replaying it
/// cannot duplicate a side effect, even for a non-idempotent operation. A
/// [`TransportFailure::Timeout`] carries no such guarantee — the request may
/// have been received and executed.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TransportFailure {
    /// DNS resolution, TCP connect, or the TLS handshake failed. The server
    /// never saw the request.
    Connect,
    /// The request was sent, but its response, or the next part of a streamed
    /// response, did not arrive within the deadline.
    Timeout,
}

impl SdkError {
    /// Remove the internal retry boundary when presenting an error to a caller.
    pub fn into_original(self) -> Self {
        match self {
            Self::ReadRetryExhausted(error) => error.into_original(),
            error => error,
        }
    }

    /// The underlying [`reqwest::Error`], whether it surfaced directly or
    /// wrapped by `reqwest_middleware`.
    ///
    /// Every client in this crate issues requests through
    /// [`reqwest_middleware::ClientWithMiddleware`], so transport failures
    /// arrive as [`SdkError::Middleware`] and never as [`SdkError::Http`].
    /// Callers that need `reqwest`'s own classification must go through here
    /// rather than matching on [`SdkError::Http`] alone.
    pub fn as_reqwest(&self) -> Option<&reqwest::Error> {
        match self {
            Self::ReadRetryExhausted(error) => error.as_reqwest(),
            Self::Http(error) => Some(error),
            Self::Middleware(reqwest_middleware::Error::Reqwest(error)) => Some(error),
            Self::EventStreamTransport(error) => Some(error),
            _ => None,
        }
    }

    /// Classify a transport-level failure, or `None` when the request reached
    /// the server and failed for some other reason.
    ///
    /// Do not attempt this by inspecting the message: `reqwest` renders every
    /// connect failure as `error sending request for url (...)`, and the words
    /// that identify it (`tcp connect error`, `dns error`) appear only in the
    /// [`std::error::Error::source`] chain.
    pub fn transport_failure(&self) -> Option<TransportFailure> {
        if let Self::ReadRetryExhausted(error) = self {
            return error.transport_failure();
        }
        if matches!(self, Self::StreamTimeout { .. }) {
            return Some(TransportFailure::Timeout);
        }
        let error = self.as_reqwest()?;
        if error.is_timeout() {
            Some(TransportFailure::Timeout)
        } else if error.is_connect() {
            Some(TransportFailure::Connect)
        } else {
            None
        }
    }

    /// The `code` from a JSON error body of the form
    /// `{"error": "...", "code": "..."}`, when the server sent one.
    ///
    /// [`SdkError::ServerError`] keeps the raw body as its message, so the code
    /// is recovered by parsing it here rather than carried as a field.
    pub fn upstream_error_code(&self) -> Option<String> {
        let Self::ServerError { message, .. } = self else {
            return None;
        };
        #[derive(serde::Deserialize)]
        struct Body {
            code: Option<String>,
        }
        serde_json::from_str::<Body>(message)
            .ok()
            .and_then(|body| body.code)
    }

    /// Whether the request provably never reached the server that would have
    /// executed it, making a replay safe even for a non-idempotent operation.
    ///
    /// True for a connect failure (see [`SdkError::transport_failure`]) and for
    /// a 5xx from the Sandbox Proxy carrying one of
    /// [`SANDBOX_PROXY_UNDELIVERED_CODES`]. Any 5xx qualifies, not only the
    /// 502/503 current proxies emit: older proxies relay the Indexify lookup's
    /// own 500/504 verbatim under `SANDBOX_UPSTREAM_ERROR`, and those requests
    /// were not forwarded either. A 4xx with one of these codes is a definitive
    /// answer the proxy passed through (a 403, a 409) and is not replayed. A
    /// timeout, or a 5xx without such a code, gives no guarantee at all.
    pub fn never_reached_server(&self) -> bool {
        if matches!(self.transport_failure(), Some(TransportFailure::Connect)) {
            return true;
        }
        let Self::ServerError { status, .. } = self else {
            return false;
        };
        if !status.is_server_error() {
            return false;
        }
        self.upstream_error_code()
            .is_some_and(|code| SANDBOX_PROXY_UNDELIVERED_CODES.contains(&code.as_str()))
    }

    /// The error's `Display` output followed by every `source()` in its chain.
    ///
    /// `reqwest` keeps the useful part of a transport failure in the chain, so
    /// reporting only `to_string()` yields `error sending request for url
    /// (...)` with no indication of whether DNS, TCP, or TLS failed.
    pub fn detail(&self) -> String {
        format_error_chain(self)
    }
}

/// Join an error's `Display` with every `source()` below it.
pub fn format_error_chain(error: &dyn std::error::Error) -> String {
    let mut message = error.to_string();
    let mut source = error.source();
    while let Some(cause) = source {
        let cause_message = cause.to_string();
        if !cause_message.is_empty() && !message.ends_with(&cause_message) {
            message.push_str(": ");
            message.push_str(&cause_message);
        }
        source = cause.source();
    }
    message
}

#[cfg(test)]
mod transport_failure_tests {
    use super::{SdkError, TransportFailure};

    /// Build the error a refused TCP connect produces, routed through
    /// `reqwest_middleware` exactly as every client in this crate does.
    async fn middleware_connect_error() -> SdkError {
        let client = reqwest_middleware::ClientBuilder::new(
            crate::http_transport::https_builder().build().unwrap(),
        )
        .build();
        // Port 1 on loopback refuses immediately; no network access needed.
        SdkError::from(client.get("http://127.0.0.1:1/x").send().await.unwrap_err())
    }

    #[tokio::test]
    async fn middleware_connect_failure_is_classified_as_connect() {
        let error = middleware_connect_error().await;

        assert_eq!(
            error.transport_failure(),
            Some(TransportFailure::Connect),
            "a connect failure through middleware must classify as Connect, not fall through \
             to the untyped arm"
        );
    }

    #[tokio::test]
    async fn middleware_connect_failure_message_alone_names_no_cause() {
        let error = middleware_connect_error().await;

        // This is the regression that made the failure undiagnosable in the
        // field: the rendered message never contains "connect" or "timeout",
        // so any substring-based classifier reports it as an internal error.
        let rendered = error.to_string().to_lowercase();
        assert!(
            !rendered.contains("connect") && !rendered.contains("timeout"),
            "reqwest's Display is expected to hide the cause; got {rendered:?}"
        );
    }

    /// A timeout must not be mistaken for a connect failure: only a connect
    /// failure proves the request never ran, and that is what gates replaying
    /// non-idempotent operations.
    #[tokio::test]
    async fn middleware_timeout_is_classified_as_timeout() {
        // A listener that accepts the connection and then never answers.
        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener");
        let address = listener.local_addr().expect("address");
        std::thread::spawn(move || {
            let mut held = Vec::new();
            while let Ok((stream, _)) = listener.accept() {
                held.push(stream);
            }
        });

        let client = reqwest_middleware::ClientBuilder::new(
            crate::http_transport::https_builder()
                .timeout(std::time::Duration::from_millis(200))
                .build()
                .unwrap(),
        )
        .build();
        let error = SdkError::from(
            client
                .get(format!("http://{address}/x"))
                .send()
                .await
                .unwrap_err(),
        );

        assert_eq!(error.transport_failure(), Some(TransportFailure::Timeout));
    }

    #[tokio::test]
    async fn detail_surfaces_the_underlying_cause() {
        let error = middleware_connect_error().await;

        let detail = error.detail().to_lowercase();
        assert!(
            detail.contains("tcp connect error"),
            "detail() must expose the source chain; got {detail:?}"
        );
    }
}

#[cfg(test)]
mod undelivered_tests {
    use super::SdkError;

    fn server_error(status: u16, body: &str) -> SdkError {
        SdkError::ServerError {
            status: reqwest::StatusCode::from_u16(status).unwrap(),
            message: body.to_string(),
        }
    }

    #[test]
    fn upstream_error_code_reads_the_proxy_body() {
        let error = server_error(
            503,
            r#"{"error":"Sandbox routing is temporarily unavailable","code":"SANDBOX_UPSTREAM_ERROR"}"#,
        );
        assert_eq!(
            error.upstream_error_code().as_deref(),
            Some("SANDBOX_UPSTREAM_ERROR")
        );
        assert_eq!(server_error(503, "plain text").upstream_error_code(), None);
        assert_eq!(
            server_error(503, r#"{"error":"no code"}"#).upstream_error_code(),
            None
        );
    }

    #[test]
    fn only_pre_forward_proxy_failures_count_as_undelivered() {
        let undelivered = [
            (503, "SANDBOX_UPSTREAM_ERROR"),
            // Pre-#2083 proxies relay the lookup's own 5xx under this code.
            (500, "SANDBOX_UPSTREAM_ERROR"),
            (504, "SANDBOX_UPSTREAM_ERROR"),
            (503, "SANDBOX_NOT_READY"),
            (502, "SANDBOX_UNREACHABLE"),
            (503, "AUTH_SERVICE_UNAVAILABLE"),
        ];
        for (status, code) in undelivered {
            let body = format!(r#"{{"error":"x","code":"{code}"}}"#);
            assert!(
                server_error(status, &body).never_reached_server(),
                "{status} {code} must be replayable"
            );
        }

        // A 4xx under the same code is a definitive answer the proxy passed
        // through from the lookup (forbidden, conflict): not replayed.
        for status in [403, 409] {
            assert!(
                !server_error(status, r#"{"error":"x","code":"SANDBOX_UPSTREAM_ERROR"}"#)
                    .never_reached_server(),
                "{status} must not be replayed"
            );
        }
        // A 503 from something else — the sandbox daemon, an ingress — may have
        // executed the request.
        assert!(!server_error(503, r#"{"error":"daemon overloaded"}"#).never_reached_server());
        assert!(
            !server_error(503, r#"{"error":"x","code":"SANDBOX_NOT_FOUND"}"#)
                .never_reached_server()
        );
        assert!(
            !server_error(404, r#"{"error":"x","code":"SANDBOX_NOT_FOUND"}"#)
                .never_reached_server()
        );
    }
}