open-net 0.4.0

A cross-platform async networking library for Rust with HTTP and WebSocket clients, network status monitoring, configurable retries, WebSocket reconnection and heartbeats, and managed client lifecycles. Reuse common networking infrastructure across applications to avoid repeated implementation and debugging, helping save development time and AI coding tokens.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
use super::{run_budget, CancellationSignal, StreamBudget};
use crate::api::error::{ErrorStage, NetError};
use bytes::{BufMut, Bytes, BytesMut};
use futures_util::future::BoxFuture;
use futures_util::{stream, FutureExt};
use reqwest::{Request, Response};
use std::future::Future;
use std::io;
use std::pin::Pin;
use std::sync::Arc;
use tokio::io::{AsyncRead, AsyncReadExt};
use tokio::sync::{mpsc, oneshot};

const CHUNK_BYTES: usize = 64 * 1024;

pub(crate) struct UploadReader {
    reader: Pin<Box<dyn AsyncRead + Send>>,
    remaining: u64,
    buffer: BytesMut,
}

impl UploadReader {
    pub(crate) fn new(reader: impl AsyncRead + Send + 'static, length: u64) -> Self {
        Self {
            reader: Box::pin(reader),
            remaining: length,
            buffer: BytesMut::new(),
        }
    }

    async fn next_chunk(&mut self) -> Result<Option<Bytes>, NetError> {
        if self.remaining == 0 {
            return Ok(None);
        }
        let limit = self.remaining.min(CHUNK_BYTES as u64) as usize;
        if self.buffer.capacity() == 0 {
            self.buffer.reserve(limit);
        }
        // Reuse the unsent portion of the allocation after a short read.
        let capacity = limit.min(self.buffer.capacity());
        let mut destination = (&mut self.buffer).limit(capacity);
        let read = self
            .reader
            .read_buf(&mut destination)
            .await
            .map_err(|error| NetError::from(error).with_stage(ErrorStage::Write))?;
        if read == 0 {
            return Err(NetError::from(io::Error::new(
                io::ErrorKind::UnexpectedEof,
                "upload reader ended before Content-Length",
            ))
            .with_stage(ErrorStage::Write));
        }
        self.remaining -= read as u64;
        Ok(Some(self.buffer.split().freeze()))
    }

    pub(super) fn attach(mut self, request: &mut Request) -> UploadTransfer {
        request
            .headers_mut()
            .insert(http::header::CONTENT_LENGTH, self.remaining.into());
        // Hyper can own the request body on a separate connection task. Keep
        // the actual reader in the request worker so cancelling that worker's
        // transfer releases even a pending reader under HTTP/2 backpressure.
        let (release, released) = oneshot::channel();
        let (pull, mut demand) =
            mpsc::channel::<oneshot::Sender<Result<(Bytes, bool), NetError>>>(1);
        let frame_state = (self.remaining != 0).then_some((pull, released));
        let frames = stream::unfold(frame_state, |state| async move {
            let (pull, released) = state?;
            let (reply, frame) = oneshot::channel();
            if pull.send(reply).await.is_ok() {
                match frame.await {
                    Ok(Ok((bytes, last))) => {
                        return Some((Ok(bytes), (!last).then_some((pull, released))))
                    }
                    Ok(Err(error)) => return Some((Err(error), None)),
                    Err(_) => {}
                }
            }
            // The reader can stop on an early rejection while the response is
            // still arriving. Signalling a short request EOF to HTTP/1 here
            // would close the connection and truncate that response.
            let _ = released.await;
            None
        });
        *request.body_mut() = Some(reqwest::Body::wrap_stream(frames));
        let pump = async move {
            while self.remaining > 0 {
                let Some(mut reply) = demand.recv().await else {
                    return Ok(SourceEnd::TransportClosed);
                };
                // Only one pull can be outstanding and no file read is spawned.
                let read = tokio::select! {
                    biased;
                    _ = reply.closed() => return Ok(SourceEnd::TransportClosed),
                    read = self.next_chunk() => read,
                };
                let bytes = match read {
                    Ok(Some(bytes)) => bytes,
                    Ok(None) => break,
                    Err(error) => {
                        let _ = reply.send(Err(error.clone()));
                        return Err(error);
                    }
                };
                if reply.send(Ok((bytes, self.remaining == 0))).is_err() {
                    return Ok(SourceEnd::TransportClosed);
                }
            }
            Ok(SourceEnd::Complete)
        };
        UploadTransfer {
            pump: Some(pump.boxed()),
            release: Some(release),
        }
    }
}

enum SourceEnd {
    Complete,
    TransportClosed,
}

impl SourceEnd {
    fn require_complete(self) -> Result<(), NetError> {
        match self {
            Self::Complete => Ok(()),
            Self::TransportClosed => Err(NetError::from(io::Error::new(
                io::ErrorKind::BrokenPipe,
                "transport stopped consuming the upload body",
            ))
            .with_stage(ErrorStage::Write)),
        }
    }
}

#[derive(Default)]
pub(super) struct UploadTransfer {
    pump: Option<BoxFuture<'static, Result<SourceEnd, NetError>>>,
    release: Option<oneshot::Sender<()>>,
}

/// Drops with the rejected response when a redirect discards it, or after
/// collection if the response is returned to the caller.
#[derive(Clone)]
pub(super) struct RejectedUpload {
    _hold: Arc<oneshot::Sender<()>>,
}

impl UploadTransfer {
    /// Drive reads concurrently with transport. Report a reader error directly,
    /// before the HTTP stack can convert it to a generic connection failure.
    pub(super) async fn headers(
        &mut self,
        sending: impl Future<Output = Result<Response, NetError>>,
    ) -> Result<Response, NetError> {
        let Some(mut pump) = self.pump.take() else {
            return sending.await;
        };
        tokio::pin!(sending);
        let result = tokio::select! {
            biased;
            end = &mut pump => {
                let end = end?;
                let response = sending.await?;
                if response.status().is_success() {
                    end.require_complete()?;
                }
                Ok(response)
            }
            result = &mut sending => {
                self.pump = Some(pump);
                result
            }
        };
        match result {
            Ok(mut response) => {
                if !response.status().is_success() && self.pump.take().is_some() {
                    if let Some(release) = self.release.take() {
                        response.extensions_mut().insert(RejectedUpload {
                            _hold: Arc::new(release),
                        });
                    }
                }
                Ok(response)
            }
            Err(error) => {
                *self = Self::default();
                Err(error)
            }
        }
    }

    /// HTTP/2 can receive a 2xx head before the peer consumes the whole
    /// request. Drive both directions to avoid a flow-control deadlock.
    pub(super) async fn finish_with<T>(
        self,
        reading: impl Future<Output = Result<T, NetError>>,
        budget: StreamBudget,
        control: &CancellationSignal,
    ) -> Result<T, NetError> {
        let Some(pump) = self.pump else {
            return reading.await;
        };
        let writing = run_budget(budget, control, ErrorStage::Write, async {
            pump.await?.require_complete()
        });
        // The response collector owns its idle timer and received head. If
        // both directions reach the same deadline while that body is still
        // pending, it reports the same failure as a regular buffered request.
        let (response, ()) = tokio::try_join!(biased; reading, writing)?;
        Ok(response)
    }
}

#[cfg(test)]
mod tests {
    use super::super::tests::check;
    use super::*;
    use crate::api::error::ErrorKind;
    use std::sync::atomic::{AtomicUsize, Ordering};
    use std::sync::Arc;
    use std::task::{Context, Poll};
    use tokio::io::ReadBuf;

    type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;

    #[derive(Default)]
    struct Reads {
        bytes: AtomicUsize,
        polls: AtomicUsize,
    }

    struct Probe {
        source: &'static [u8],
        failure: Option<io::ErrorKind>,
        reads: Arc<Reads>,
    }

    impl AsyncRead for Probe {
        fn poll_read(
            mut self: Pin<&mut Self>,
            cx: &mut Context<'_>,
            buf: &mut ReadBuf<'_>,
        ) -> Poll<io::Result<()>> {
            self.reads.polls.fetch_add(1, Ordering::Relaxed);
            if self.source.is_empty() {
                if let Some(kind) = self.failure {
                    return Poll::Ready(Err(kind.into()));
                }
            }
            let before = buf.filled().len();
            let result = Pin::new(&mut self.source).poll_read(cx, buf);
            self.reads
                .bytes
                .fetch_add(buf.filled().len() - before, Ordering::Relaxed);
            result
        }
    }

    #[tokio::test]
    async fn reads_only_the_declared_prefix_including_zero() -> TestResult {
        for (len, prefix) in [(0, None), (3, Some(&b"abc"[..]))] {
            let reads = Arc::new(Reads::default());
            let probe = Probe {
                source: b"abc-more",
                reads: reads.clone(),
                failure: None,
            };
            let mut upload = UploadReader::new(probe, len);
            check(
                upload.next_chunk().await?.as_deref() == prefix,
                "wrong prefix",
            )?;
            check(upload.next_chunk().await?.is_none(), "extra upload bytes")?;
            check(
                reads.bytes.load(Ordering::Relaxed) as u64 == len,
                "source overread",
            )?;
            check(
                reads.polls.load(Ordering::Relaxed) == usize::from(len != 0),
                "extra reader poll",
            )?;
        }
        Ok(())
    }

    #[tokio::test]
    async fn short_eof_and_source_errors_preserve_the_io_category() -> TestResult {
        for (failure, expected) in [
            (None, io::ErrorKind::UnexpectedEof),
            (
                Some(io::ErrorKind::PermissionDenied),
                io::ErrorKind::PermissionDenied,
            ),
        ] {
            let probe = Probe {
                source: b"abc",
                reads: Arc::new(Reads::default()),
                failure,
            };
            let mut upload = UploadReader::new(probe, 8);
            check(
                upload.next_chunk().await?.as_deref() == Some(&b"abc"[..]),
                "lost initial bytes",
            )?;
            check(
                upload.next_chunk().await.is_err_and(|error| {
                    error.kind() == ErrorKind::Io
                        && error.context().stage == Some(ErrorStage::Write)
                        && error.io_kind() == Some(expected)
                }),
                "lost local reader error",
            )?;
        }
        Ok(())
    }

    #[tokio::test]
    async fn huge_lengths_use_bounded_chunks() -> TestResult {
        let mut upload = UploadReader::new(tokio::io::repeat(1), u64::MAX);
        for _ in 0..2 {
            check(
                upload
                    .next_chunk()
                    .await?
                    .is_some_and(|bytes| bytes.len() == CHUNK_BYTES),
                "wrong read capacity for large Content-Length",
            )?;
        }
        Ok(())
    }

    #[tokio::test(start_paused = true)]
    async fn expired_attempt_after_headers_uses_the_pending_response_read() -> TestResult {
        use super::super::read_response;
        use crate::api::http::{HttpRequestId, TimeoutPhase};
        use std::time::Duration;
        use tokio::time::Instant;

        let duration = Duration::from_secs(1);
        let budget = StreamBudget::default().with_phase(
            Instant::now(),
            Some(duration),
            TimeoutPhase::Attempt,
        )?;
        let control = Arc::new(CancellationSignal::new());
        let response = http::Response::builder()
            .status(http::StatusCode::CREATED)
            .header("x-test-head", "ready")
            .body(reqwest::Body::wrap_stream(stream::pending::<
                Result<Bytes, io::Error>,
            >()))?;
        #[cfg(feature = "http-client-http2")]
        let http2 = crate::api::http::Http2Config::new();
        let reading = read_response(
            response.into(),
            1,
            HttpRequestId(1),
            1024,
            control.clone(),
            #[cfg(feature = "http-client-http2")]
            &http2,
            budget,
            None,
        );
        // The upload has delivered its last chunk. The response headers arrived
        // before expiry, but the body has not been collected when the timer wins.
        let transfer = UploadTransfer {
            pump: Some(async { Ok(SourceEnd::Complete) }.boxed()),
            ..UploadTransfer::default()
        };
        tokio::time::advance(duration).await;
        let error = transfer
            .finish_with(reading, budget, &control)
            .await
            .err()
            .ok_or("expired response body unexpectedly succeeded")?;
        check(
            error.timeout_phase() == Some(TimeoutPhase::Attempt)
                && error.context().stage == Some(ErrorStage::Receive)
                && error.http_response_head().is_some_and(|head| {
                    head.status == http::StatusCode::CREATED
                        && head
                            .headers
                            .get("x-test-head")
                            .is_some_and(|value| value == "ready")
                }),
            "upload replaced the response read timeout or its received headers",
        )
    }

    #[tokio::test(start_paused = true)]
    async fn response_collected_before_a_source_stall_keeps_the_write_timeout() -> TestResult {
        use crate::api::http::TimeoutPhase;
        use std::time::Duration;
        use tokio::time::Instant;

        let budget = StreamBudget::total(Instant::now(), Some(Duration::from_secs(1)))?;
        let transfer = UploadTransfer {
            pump: Some(std::future::pending().boxed()),
            ..UploadTransfer::default()
        };
        let error = transfer
            .finish_with(async { Ok(()) }, budget, &CancellationSignal::new())
            .await
            .err()
            .ok_or("stalled upload was reported as complete")?;
        check(
            error.kind() == ErrorKind::DeadlineExceeded
                && error.timeout_phase() == Some(TimeoutPhase::Total)
                && error.context().stage == Some(ErrorStage::Write)
                && error.http_response_head().is_none(),
            "a pending reader was misclassified as a response-body failure",
        )
    }
}