eggfetch-core 0.3.0

Async-first Rust HTTP client engine (tokio + hyper) with connection pooling, phase-aware timeouts, TLS, proxies, and compression
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
//! Incoming response type.
//!
//! A `Response` holds the HTTP status, version, headers, final URL, and a
//! body handle. The body can be consumed in two ways:
//!
//! - **Buffered**: `bytes()` or `text()` collects the entire body.
//! - **Streaming**: `bytes_stream()` yields decoded body chunks incrementally;
//!   `raw_bytes_stream()` selects encoded chunks for compressed responses.
//!
//! # Body ownership
//!
//! The body can be consumed exactly once. After calling `bytes()`,
//! `text()`, `bytes_stream()`, or `raw_bytes_stream()`, further reads return
//! an error.

use std::pin::Pin;
use std::task::{Context, Poll};

use bytes::{Bytes, BytesMut};
use futures_core::Stream;
use http::{HeaderMap, StatusCode, Version};
use url::Url;

use crate::body::{BoxBytesStream, ResponseBody, SharedTrailers};
use crate::error::Result;
use crate::network_stream::NetworkStream;

/// A metadata-only snapshot of a redirect response.
///
/// History entries intentionally omit the response body. This makes it
/// structurally impossible for redirect history to retain active body
/// streams or pool permits.
#[derive(Clone)]
pub struct HistoryEntry {
    status: StatusCode,
    version: Version,
    headers: HeaderMap,
    url: Url,
    reason_phrase: String,
}

impl std::fmt::Debug for HistoryEntry {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("HistoryEntry")
            .field("status", &self.status)
            .field("version", &self.version)
            .field("headers", &redacted_headers(&self.headers))
            .field("url", &redacted_url(&self.url))
            .field("reason_phrase", &self.reason_phrase)
            .finish()
    }
}

impl HistoryEntry {
    /// Create a history entry from a response that has already been
    /// drained (body consumed).
    ///
    /// Only available with the `redirects` feature; the lean profile
    /// constructs no redirect history.
    #[cfg(feature = "redirects")]
    pub(crate) fn from_response(response: &Response) -> Self {
        let reason = response
            .wire_reason_phrase
            .clone()
            .or_else(|| response.status.canonical_reason().map(str::to_owned))
            .unwrap_or_default();
        Self {
            status: response.status,
            version: response.version,
            headers: response.headers.clone(),
            url: response.url.clone(),
            reason_phrase: reason,
        }
    }

    /// Returns the HTTP status code.
    #[must_use]
    pub fn status(&self) -> StatusCode {
        self.status
    }

    /// Returns the HTTP version.
    #[must_use]
    pub fn version(&self) -> Version {
        self.version
    }

    /// Returns the response headers.
    #[must_use]
    pub fn headers(&self) -> &HeaderMap {
        &self.headers
    }

    /// Returns the URL.
    #[must_use]
    pub fn url(&self) -> &Url {
        &self.url
    }

    /// Returns the reason phrase.
    #[must_use]
    pub fn reason_phrase(&self) -> &str {
        &self.reason_phrase
    }
}

/// An HTTP response.
///
/// # Trailers
///
/// HTTP trailers (HTTP/1.1 chunked trailers, HTTP/2 trailing HEADERS, and
/// HTTP/3 trailing headers where the h3 crate exposes them) are captured
/// without buffering the body. The body stream yields data frames exactly
/// as before; when the trailing-headers frame arrives it is stored and the
/// stream ends normally. Use [`Self::trailers`] after fully consuming the
/// body (`bytes()`, `bytes_stream()` to EOF) to retrieve them.
///
/// Lifecycle: `trailers()` returns `None` until the body has advanced far
/// enough for trailers to arrive, when the response carried no trailers,
/// or when a body error occurred before trailers (errors remain body
/// errors, never fabricated trailer state). Partial consumption or early
/// drop never reports trailers and releases the pool permit without
/// waiting for them. Read timeouts apply while waiting for trailers at
/// the body boundary; the absolute native `total` deadline spans the full
/// lifecycle through body EOF/trailers, never resets per trailer wait, and
/// reports `Total` when polled after expiry. Dropping before EOF remains
/// ordinary cancellation.
pub struct Response {
    status: StatusCode,
    version: Version,
    headers: HeaderMap,
    url: Url,
    /// Original wire value retained for compatibility adapters that expose
    /// response metadata after automatic decompression.
    wire_content_encoding: Option<String>,
    /// Original wire value retained for compatibility adapters that expose
    /// response metadata after automatic decompression.
    wire_content_length: Option<String>,
    /// Original wire reason phrase from the HTTP status line.
    ///
    /// HTTP/1.x responses include a reason phrase after the status code.
    /// This field preserves the original wire bytes so compatibility
    /// adapters can expose the exact phrase rather than deriving one from
    /// the status code alone. HTTP/2 has no wire reason phrase; this
    /// field will be `None` for H2 responses.
    wire_reason_phrase: Option<String>,
    pub(crate) body: ResponseBody,
    history: Vec<HistoryEntry>,
    /// Shared trailer store populated as the body stream advances.
    ///
    /// Cloned into the body stream at construction so `trailers()` stays
    /// available after `bytes()` collection. `None` until trailers arrive.
    trailers: SharedTrailers,
    /// Optional network stream handle for connection metadata and
    /// upgraded-connection IO. Set for responses where the underlying
    /// transport metadata is available:
    ///
    /// - **101 Switching Protocols**: set to `NetworkStream::Upgraded`
    ///   with full IO access.
    /// - **Successful CONNECT (200)**: set to `NetworkStream::Upgraded`
    ///   when the tunnel is established (handled in the proxy path).
    /// - **Ordinary HTTP/1.1 and HTTP/2 responses**: always `None`.
    ///   Hyper's connection pool retains ownership of the socket and
    ///   does not expose per-response socket metadata. Raw IO would
    ///   corrupt pool state. This is a documented bounded difference
    ///   from httpcore, which owns its connections directly.
    network_stream: Option<NetworkStream>,
}

impl std::fmt::Debug for Response {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Response")
            .field("status", &self.status)
            .field("version", &self.version)
            .field("headers", &redacted_headers(&self.headers))
            .field("url", &redacted_url(&self.url))
            .field("wire_content_encoding", &self.wire_content_encoding)
            .field("wire_content_length", &self.wire_content_length)
            .field("wire_reason_phrase", &self.wire_reason_phrase)
            .field("body", &self.body)
            .field("history", &self.history)
            .field(
                "trailers",
                &self.trailers.get().map(|h| redacted_headers(&h)),
            )
            .field(
                "network_stream",
                &self.network_stream.as_ref().map(|_| "..."),
            )
            .finish()
    }
}

fn redacted_headers(headers: &HeaderMap) -> HeaderMap {
    crate::redact::redact_headers(headers)
}

fn redacted_url(url: &Url) -> String {
    crate::redact::redact_url(url)
}

impl Response {
    /// Create a new response (crate-internal).
    pub(crate) fn new(
        status: StatusCode,
        version: Version,
        headers: HeaderMap,
        url: Url,
        body: ResponseBody,
    ) -> Self {
        // Non-UTF8 wire values are dropped here (defensible: headers are
        // opaque bytes); emit a trace event so corrupt-wire debugging has a
        // signal instead of a silent absence. Laziness is load-bearing: the
        // trace event must fire only when the value is dropped, so eager
        // evaluation is not equivalent here.
        #[allow(
            clippy::unnecessary_lazy_evaluations,
            reason = "the trace event must fire only when the value is dropped, so the lazy closure is not equivalent to eager evaluation"
        )]
        let wire_content_encoding = headers
            .get("content-encoding")
            .and_then(|value| {
                value.to_str().ok().or_else(|| {
                    #[cfg(feature = "tracing")]
                    tracing::debug!("eggfetch: dropped non-UTF8 content-encoding");
                    None
                })
            })
            .map(ToOwned::to_owned);
        // Laziness is load-bearing here as above: eager evaluation would
        // emit the drop trace event even for valid values.
        #[allow(
            clippy::unnecessary_lazy_evaluations,
            reason = "the trace event must fire only when the value is dropped, so the lazy closure is not equivalent to eager evaluation"
        )]
        let wire_content_length = headers
            .get("content-length")
            .and_then(|value| {
                value.to_str().ok().or_else(|| {
                    #[cfg(feature = "tracing")]
                    tracing::debug!("eggfetch: dropped non-UTF8 content-length");
                    None
                })
            })
            .map(ToOwned::to_owned);
        Self {
            status,
            version,
            headers,
            url,
            wire_content_encoding,
            wire_content_length,
            wire_reason_phrase: None,
            body,
            history: Vec::new(),
            trailers: SharedTrailers::new(),
            network_stream: None,
        }
    }

    /// Replace the body of this response. Crate-internal: used to attach
    /// a leased body after pool acquisition.
    pub(crate) fn set_body(&mut self, body: ResponseBody) {
        self.body = body;
    }

    /// Attach the shared trailer store linked to the body stream.
    ///
    /// Transports create one [`SharedTrailers`], hand a clone to the
    /// streaming adapter, and install the same instance here so
    /// [`Self::trailers`] observes trailers populated during streaming.
    pub(crate) fn set_trailers(&mut self, trailers: SharedTrailers) {
        self.trailers = trailers;
    }

    /// Returns received HTTP trailers, if any.
    ///
    /// Returns `None` until the body has been fully consumed and trailers
    /// have arrived, when the response carried no trailers, or when a body
    /// error occurred before trailers. Duplicate/multi-value trailing
    /// headers are preserved in the returned [`HeaderMap`].
    ///
    /// This never buffers the body: drive the body to EOF via `bytes()` or
    /// `bytes_stream()` first, then call this method.
    #[must_use]
    pub fn trailers(&self) -> Option<HeaderMap> {
        self.trailers.get()
    }

    /// Returns the HTTP status code.
    #[must_use]
    pub fn status(&self) -> StatusCode {
        self.status
    }

    /// Returns the HTTP version.
    #[must_use]
    pub fn version(&self) -> Version {
        self.version
    }

    /// Returns the response headers.
    #[must_use]
    pub fn headers(&self) -> &HeaderMap {
        &self.headers
    }

    /// Returns a mutable reference to the response headers.
    pub fn headers_mut(&mut self) -> &mut HeaderMap {
        &mut self.headers
    }

    /// Returns the original wire `Content-Encoding` value, if present.
    ///
    /// Automatic decompression may remove this value from [`Self::headers`];
    /// compatibility adapters can use this read-only snapshot to preserve
    /// wire metadata without changing the core decoded-header policy.
    #[must_use]
    pub fn wire_content_encoding(&self) -> Option<&str> {
        self.wire_content_encoding.as_deref()
    }

    /// Returns the original wire `Content-Length` value, if present.
    ///
    /// The value describes encoded wire bytes and is never derived from the
    /// decompressed response body.
    #[must_use]
    pub fn wire_content_length(&self) -> Option<&str> {
        self.wire_content_length.as_deref()
    }

    /// Returns the declared wire `Content-Length` as an unsigned integer.
    ///
    /// This parses only the original header value. It never reports the
    /// collected or decompressed body length and returns `None` when the
    /// header is absent or invalid.
    #[must_use]
    pub fn content_length(&self) -> Option<u64> {
        self.wire_content_length.as_deref()?.parse().ok()
    }

    /// Returns the original wire reason phrase, if present.
    ///
    /// HTTP/1.x responses include a reason phrase after the status code.
    /// This preserves the original wire bytes rather than deriving from
    /// the status code alone. HTTP/2 has no wire reason phrase.
    #[must_use]
    pub fn wire_reason_phrase(&self) -> Option<&str> {
        self.wire_reason_phrase.as_deref()
    }

    /// Set the wire reason phrase for this response.
    ///
    /// Used by transports that parse the HTTP/1.x status line directly
    /// and can extract the original reason phrase.
    #[allow(
        dead_code,
        reason = "the wire reason phrase is only set by the proxy and CONNECT transports, which are compiled out when the proxy feature is disabled"
    )]
    pub(crate) fn set_wire_reason_phrase(&mut self, reason: Option<String>) {
        self.wire_reason_phrase = reason;
    }

    /// Returns the final URL of the response (after any redirects).
    #[must_use]
    pub fn url(&self) -> &Url {
        &self.url
    }

    /// Returns the redirect history (prior responses in order).
    #[must_use]
    pub fn history(&self) -> &[HistoryEntry] {
        &self.history
    }

    /// Returns a mutable reference to the redirect history.
    pub fn history_mut(&mut self) -> &mut Vec<HistoryEntry> {
        &mut self.history
    }

    /// Set the redirect history.
    ///
    /// Only available with the `redirects` feature; the lean profile leaves
    /// history empty.
    #[cfg(feature = "redirects")]
    pub(crate) fn set_history(&mut self, history: Vec<HistoryEntry>) {
        self.history = history;
    }

    /// Returns `true` if the status code indicates success (2xx).
    #[must_use]
    pub fn is_success(&self) -> bool {
        self.status.is_success()
    }

    /// Returns a reference to the response body.
    #[must_use]
    pub fn body(&self) -> &ResponseBody {
        &self.body
    }

    /// Returns a mutable reference to the response body.
    pub fn body_mut(&mut self) -> &mut ResponseBody {
        &mut self.body
    }

    /// Consume the response and return the body bytes.
    ///
    /// For buffered bodies, this is O(1). For streaming bodies, this
    /// collects all chunks.
    ///
    /// # Errors
    ///
    /// Returns an error if the body has already been consumed or if a
    /// stream chunk fails.
    pub async fn bytes(&mut self) -> Result<Bytes> {
        self.body.bytes().await
    }

    /// Consume the response and return the body as a UTF-8 string.
    ///
    /// # Errors
    ///
    /// Returns an error if the body is not valid UTF-8 or has already
    /// been consumed.
    pub async fn text(&mut self) -> Result<String> {
        self.body.text().await
    }

    /// Consume and deserialize the response body as JSON.
    ///
    /// This follows [`Self::bytes`] single-consumption and decoded-body-limit
    /// semantics. Media type is not checked; calling this method is explicit
    /// caller intent. The body is parsed directly from bytes without a lossy
    /// UTF-8 conversion.
    ///
    /// # Errors
    ///
    /// Returns a body error when the response cannot be consumed, or
    /// [`crate::Error::JsonDeserialize`] when the bytes are not valid JSON.
    #[cfg(feature = "json")]
    pub async fn json<T: serde::de::DeserializeOwned>(&mut self) -> Result<T> {
        let bytes = self.bytes().await?;
        serde_json::from_slice(&bytes).map_err(|error| {
            crate::Error::JsonDeserialize(format!(
                "invalid JSON at line {}, column {}",
                error.line(),
                error.column()
            ))
        })
    }

    /// Consume the response body and return a stream of byte chunks.
    ///
    /// For buffered bodies, yields the entire body as a single chunk.
    /// For streaming bodies, yields chunks incrementally.
    ///
    /// # Errors
    ///
    /// Returns an error if the body has already been consumed.
    pub fn bytes_stream(&mut self) -> Result<BoxBytesStream> {
        self.body.bytes_stream()
    }

    /// Consume the response body and return encoded byte chunks.
    ///
    /// Compressed streaming responses select the original encoded transport
    /// body. Raw and decoded body selection are mutually exclusive and
    /// single-consumption operations.
    ///
    /// # Errors
    ///
    /// Returns an error if the response body has already been consumed.
    pub fn raw_bytes_stream(&mut self) -> Result<BoxBytesStream> {
        self.body.raw_bytes_stream()
    }

    /// Consume the response body and return a stream of text lines.
    ///
    /// Each line is decoded as UTF-8 without the trailing newline.
    /// Incomplete lines at chunk boundaries are buffered until the
    /// next chunk completes them.
    ///
    /// # Errors
    ///
    /// Returns an error if the body has already been consumed or if a
    /// chunk contains invalid UTF-8.
    pub fn text_lines(&mut self) -> Result<impl Stream<Item = Result<String>>> {
        let byte_stream = self.bytes_stream()?;
        Ok(LineStream::new(byte_stream))
    }

    /// Returns a reference to the network stream handle, if available.
    ///
    /// For upgraded connections (101/CONNECT), this provides full
    /// IO access. For ordinary pooled connections, this provides
    /// read-only metadata.
    #[must_use]
    pub fn network_stream(&self) -> Option<&NetworkStream> {
        self.network_stream.as_ref()
    }

    /// Returns a mutable reference to the network stream handle.
    pub fn network_stream_mut(&mut self) -> Option<&mut NetworkStream> {
        self.network_stream.as_mut()
    }

    /// Take the network stream out of the response, leaving `None` in
    /// its place. This is the canonical way to obtain ownership of an
    /// upgraded stream while still keeping the rest of the response.
    #[must_use]
    pub fn take_network_stream(&mut self) -> Option<NetworkStream> {
        self.network_stream.take()
    }

    /// Set the network stream handle for this response.
    pub(crate) fn set_network_stream(&mut self, stream: NetworkStream) {
        self.network_stream = Some(stream);
    }

    /// Consume the response and return the network stream, if any.
    ///
    /// This is the primary way to obtain ownership of an upgraded
    /// stream (101 Switching Protocols or successful CONNECT).
    #[must_use]
    pub fn into_network_stream(self) -> Option<NetworkStream> {
        self.network_stream
    }
}

/// Stream adapter that splits a byte stream into text lines.
struct LineStream {
    stream: BoxBytesStream,
    buffer: BytesMut,
}

const MAX_LINE_LENGTH: usize = 1024 * 1024;

impl LineStream {
    fn new(stream: BoxBytesStream) -> Self {
        Self {
            stream,
            buffer: BytesMut::new(),
        }
    }
}

impl Stream for LineStream {
    type Item = Result<String>;

    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
        loop {
            // Fast path: emit a line already sitting in the buffer. The
            // length cap applies here too — a single chunk may carry an
            // over-long line piggybacked behind a valid one.
            if let Some(pos) = self.buffer.iter().position(|&b| b == b'\n') {
                if pos > MAX_LINE_LENGTH {
                    return Poll::Ready(Some(Err(crate::error::Error::Body(format!(
                        "response line exceeded maximum length of {MAX_LINE_LENGTH} bytes"
                    )))));
                }
                let line_bytes = self.buffer.split_to(pos + 1);
                let mut line = &line_bytes[..line_bytes.len() - 1]; // strip \n
                if line.ends_with(b"\r") {
                    line = &line[..line.len() - 1];
                }
                let line = std::str::from_utf8(line)
                    .map(str::to_owned)
                    .map_err(|e| crate::error::Error::Body(e.to_string()))?;
                return Poll::Ready(Some(Ok(line)));
            }

            // No newline yet; a partially assembled line must stay bounded.
            if self.buffer.len() > MAX_LINE_LENGTH {
                return Poll::Ready(Some(Err(crate::error::Error::Body(format!(
                    "response line exceeded maximum length of {MAX_LINE_LENGTH} bytes"
                )))));
            }

            // Pull more data. Ready chunks are drained in this loop rather
            // than rescheduling through the executor per chunk.
            match Pin::new(&mut self.stream).poll_next(cx) {
                Poll::Ready(Some(Ok(chunk))) => {
                    if chunk.is_empty() {
                        // Empty chunks carry no data. Ignore them and let the
                        // upstream stream determine whether another wake is
                        // needed.
                        continue;
                    }
                    // Bound the buffer before extending: a single chunk of
                    // size `MAX+1` without a newline would otherwise push the
                    // buffer to `MAX + chunk` before the next loop's limit
                    // check. If the combined buffer would exceed `MAX` and
                    // there is no newline within the first `MAX` bytes, reject
                    // immediately without allocating the oversize tail.
                    if self.buffer.len().saturating_add(chunk.len()) > MAX_LINE_LENGTH {
                        let remaining = MAX_LINE_LENGTH.saturating_sub(self.buffer.len());
                        // `buffer` is known to contain no newline (otherwise
                        // we would have emitted at the top of the loop), so
                        // only need to inspect the prefix of `chunk` that fits
                        // within the limit.
                        let prefix = &chunk[..remaining.min(chunk.len())];
                        if !prefix.contains(&b'\n') {
                            // Also check if chunk itself contains an overlong
                            // line that straddles the boundary: if the first
                            // newline in the full chunk is beyond `MAX`, the
                            // first line is already overlong.
                            if let Some(pos) = chunk.iter().position(|&b| b == b'\n') {
                                if self.buffer.len() + pos > MAX_LINE_LENGTH {
                                    return Poll::Ready(Some(Err(
                                        crate::error::Error::Body(format!(
                                            "response line exceeded maximum length of {MAX_LINE_LENGTH} bytes"
                                        )),
                                    )));
                                }
                            } else {
                                return Poll::Ready(Some(Err(
                                    crate::error::Error::Body(format!(
                                        "response line exceeded maximum length of {MAX_LINE_LENGTH} bytes"
                                    )),
                                )));
                            }
                        }
                    }
                    self.buffer.extend_from_slice(&chunk);
                }
                Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))),
                Poll::Ready(None) => {
                    // Stream ended; flush remaining buffer as a final line.
                    if self.buffer.is_empty() {
                        return Poll::Ready(None);
                    }
                    let remaining = std::mem::take(&mut self.buffer);
                    if remaining.len() > MAX_LINE_LENGTH {
                        return Poll::Ready(Some(Err(crate::error::Error::Body(format!(
                            "response line exceeded maximum length of {MAX_LINE_LENGTH} bytes"
                        )))));
                    }
                    let mut line = &remaining[..remaining.len()];
                    if line.ends_with(b"\r") {
                        line = &line[..line.len() - 1];
                    }
                    let line = std::str::from_utf8(line)
                        .map(str::to_owned)
                        .map_err(|e| crate::error::Error::Body(e.to_string()))?;
                    return Poll::Ready(Some(Ok(line)));
                }
                Poll::Pending => return Poll::Pending,
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use http::HeaderValue;

    #[test]
    fn response_debug_redacts_sensitive_headers_and_url_parts() {
        let mut headers = HeaderMap::new();
        headers.insert("authorization", HeaderValue::from_static("Bearer secret"));
        headers.insert("set-cookie", HeaderValue::from_static("session=secret"));
        let response = Response::new(
            StatusCode::FOUND,
            Version::HTTP_11,
            headers,
            Url::parse("https://user:pass@example.com/path?token=secret#fragment").unwrap(),
            ResponseBody::buffered(Bytes::new()),
        );
        let rendered = format!("{response:?}");
        assert!(!rendered.contains("secret"));
        assert!(!rendered.contains("user:pass"));
        assert!(rendered.contains("https://example.com/path"));
    }

    #[test]
    fn response_retains_wire_content_metadata_without_changing_headers() {
        let mut headers = HeaderMap::new();
        headers.insert("content-encoding", HeaderValue::from_static("gzip"));
        headers.insert("content-length", HeaderValue::from_static("42"));
        let response = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            headers,
            Url::parse("http://example.com").unwrap(),
            ResponseBody::buffered(Bytes::new()),
        );

        assert_eq!(response.wire_content_encoding(), Some("gzip"));
        assert_eq!(response.wire_content_length(), Some("42"));
        assert_eq!(
            response.headers().get("content-encoding"),
            Some(&HeaderValue::from_static("gzip"))
        );
        assert_eq!(
            response.headers().get("content-length"),
            Some(&HeaderValue::from_static("42"))
        );
        assert_eq!(response.content_length(), Some(42));
    }

    #[test]
    fn invalid_wire_content_length_is_not_inferred() {
        let mut headers = HeaderMap::new();
        headers.insert("content-length", HeaderValue::from_static("not-a-number"));
        let response = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            headers,
            Url::parse("http://example.com").unwrap(),
            ResponseBody::buffered(Bytes::from_static(b"1234")),
        );
        assert_eq!(response.content_length(), None);
    }

    #[cfg(feature = "json")]
    #[tokio::test]
    async fn response_json_deserializes_once_and_classifies_parse_errors() {
        let mut response = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::buffered(Bytes::from_static(br#"{"key":"value"}"#)),
        );
        let value: serde_json::Value = response.json().await.unwrap();
        assert_eq!(value["key"], "value");
        assert!(matches!(response.bytes().await, Err(crate::Error::Body(_))));

        let mut invalid = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::buffered(Bytes::from_static(b"{not-json")),
        );
        let error = invalid.json::<serde_json::Value>().await.unwrap_err();
        assert_eq!(error.kind(), "json_deserialize");
        assert!(!error.to_string().contains("not-json"));
    }
    use futures_util::StreamExt;

    #[tokio::test]
    async fn response_bytes_and_text() {
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::buffered(Bytes::from("hello")),
        );
        assert!(resp.is_success());
        let text = resp.text().await.unwrap();
        assert_eq!(text, "hello");
    }

    #[tokio::test]
    async fn response_streaming_bytes_collect() {
        let chunks = vec![Ok(Bytes::from("hello ")), Ok(Bytes::from("world"))];
        let stream = Box::pin(futures_util::stream::iter(chunks));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        let bytes = resp.bytes().await.unwrap();
        assert_eq!(bytes, "hello world");
    }

    #[tokio::test]
    async fn response_text_lines_rejects_unbounded_line() {
        let stream = Box::pin(futures_util::stream::iter(vec![Ok(Bytes::from(vec![
            b'a'; MAX_LINE_LENGTH + 1
        ]))]));
        let mut response = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        let mut lines = response.text_lines().unwrap();
        let error = lines.next().await.unwrap().unwrap_err();
        assert!(matches!(error, crate::error::Error::Body(_)));
    }

    #[tokio::test]
    async fn response_text_lines_rejects_overlong_line_after_valid_line() {
        // A single chunk carrying a valid line followed by an over-long
        // line must not smuggle the second line past MAX_LINE_LENGTH via
        // the buffered fast path.
        let mut payload = b"ok\n".to_vec();
        payload.extend_from_slice(&vec![b'a'; MAX_LINE_LENGTH + 1]);
        payload.push(b'\n');
        let stream = Box::pin(futures_util::stream::iter(vec![Ok(Bytes::from(payload))]));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        let mut lines = resp.text_lines().unwrap();
        assert_eq!(lines.next().await.unwrap().unwrap(), "ok");
        let error = lines.next().await.unwrap().unwrap_err();
        assert!(matches!(error, crate::error::Error::Body(_)));
    }

    #[tokio::test]
    async fn response_text_lines_rejects_unbounded_terminal_line() {
        // The terminal flush path (`Poll::Ready(None)`) must enforce
        // `MAX_LINE_LENGTH` even when the over-long buffer never sees a
        // trailing newline.
        let stream = Box::pin(futures_util::stream::iter(vec![Ok(Bytes::from(vec![
            b'a'; MAX_LINE_LENGTH + 1
        ]))]));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        let mut lines = resp.text_lines().unwrap();
        let error = lines.next().await.unwrap().unwrap_err();
        assert!(matches!(error, crate::error::Error::Body(_)));
    }

    #[tokio::test]
    async fn response_bytes_stream_yields_chunks() {
        let chunks = vec![Ok(Bytes::from("a")), Ok(Bytes::from("b"))];
        let stream = Box::pin(futures_util::stream::iter(chunks));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        let mut stream = resp.bytes_stream().unwrap();
        assert_eq!(stream.next().await.unwrap().unwrap(), "a");
        assert_eq!(stream.next().await.unwrap().unwrap(), "b");
        assert!(stream.next().await.is_none());
    }

    #[tokio::test]
    async fn response_text_lines_basic() {
        let chunks = vec![Ok(Bytes::from("line1\nline2\nline3\n"))];
        let stream = Box::pin(futures_util::stream::iter(chunks));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        let mut lines = resp.text_lines().unwrap();
        assert_eq!(lines.next().await.unwrap().unwrap(), "line1");
        assert_eq!(lines.next().await.unwrap().unwrap(), "line2");
        assert_eq!(lines.next().await.unwrap().unwrap(), "line3");
        assert!(lines.next().await.is_none());
    }

    #[tokio::test]
    async fn response_text_lines_crlf() {
        let chunks = vec![Ok(Bytes::from("line1\r\nline2\r\n"))];
        let stream = Box::pin(futures_util::stream::iter(chunks));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        let mut lines = resp.text_lines().unwrap();
        assert_eq!(lines.next().await.unwrap().unwrap(), "line1");
        assert_eq!(lines.next().await.unwrap().unwrap(), "line2");
        assert!(lines.next().await.is_none());
    }

    #[tokio::test]
    async fn response_text_lines_across_chunks() {
        let chunks = vec![
            Ok(Bytes::from("hel")),
            Ok(Bytes::from("lo\nwor")),
            Ok(Bytes::from("ld\n")),
        ];
        let stream = Box::pin(futures_util::stream::iter(chunks));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        let mut lines = resp.text_lines().unwrap();
        assert_eq!(lines.next().await.unwrap().unwrap(), "hello");
        assert_eq!(lines.next().await.unwrap().unwrap(), "world");
        assert!(lines.next().await.is_none());
    }

    #[tokio::test]
    async fn response_text_lines_no_trailing_newline() {
        let chunks = vec![Ok(Bytes::from("hello"))];
        let stream = Box::pin(futures_util::stream::iter(chunks));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        let mut lines = resp.text_lines().unwrap();
        assert_eq!(lines.next().await.unwrap().unwrap(), "hello");
        assert!(lines.next().await.is_none());
    }

    #[test]
    fn trailers_absent_initially() {
        let resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::buffered(Bytes::new()),
        );
        assert!(resp.trailers().is_none());
    }

    #[test]
    fn trailers_preserve_duplicate_multi_value() {
        use crate::body::SharedTrailers;
        let shared = SharedTrailers::new();
        let mut map = HeaderMap::new();
        map.append("x-trailer", HeaderValue::from_static("a"));
        map.append("x-trailer", HeaderValue::from_static("b"));
        map.insert("x-single", HeaderValue::from_static("v"));
        shared.store(map);
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::buffered(Bytes::new()),
        );
        resp.set_trailers(shared);
        let trailers = resp.trailers().expect("trailers present");
        let values: Vec<_> = trailers
            .get_all("x-trailer")
            .iter()
            .map(|v| v.to_str().unwrap().to_owned())
            .collect();
        assert_eq!(values, vec!["a", "b"]);
        assert_eq!(trailers.get("x-single").unwrap(), "v");
    }

    #[tokio::test]
    async fn trailers_available_after_bytes_collection() {
        use crate::body::SharedTrailers;
        // Simulate a transport that populates trailers when the body is
        // driven to EOF (same contract as `wrap_incoming`).
        let shared = SharedTrailers::new();
        let shared_for_stream = shared.clone();
        let stream = Box::pin(futures_util::stream::unfold(0, move |step| {
            let shared_for_stream = shared_for_stream.clone();
            async move {
                if step == 0 {
                    Some((Ok(Bytes::from("hello")), 1))
                } else if step == 1 {
                    let mut map = HeaderMap::new();
                    map.insert("x-done", HeaderValue::from_static("yes"));
                    shared_for_stream.store(map);
                    None
                } else {
                    None
                }
            }
        }));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        resp.set_trailers(shared);
        assert!(
            resp.trailers().is_none(),
            "partial consumption reports None"
        );
        let bytes = resp.bytes().await.unwrap();
        assert_eq!(bytes, "hello");
        assert_eq!(
            resp.trailers().unwrap().get("x-done").unwrap(),
            "yes",
            "trailers available after body completion"
        );
    }

    #[tokio::test]
    async fn partial_consumption_does_not_falsely_report_trailers() {
        let stream = Box::pin(futures_util::stream::iter(vec![Ok(Bytes::from("a"))]));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        // No trailers wired; even after partial read there must be none.
        let mut s = resp.bytes_stream().unwrap();
        assert_eq!(s.next().await.unwrap().unwrap(), "a");
        assert!(resp.trailers().is_none());
    }

    #[tokio::test]
    async fn body_error_before_trailers_leaves_no_trailers() {
        use crate::body::SharedTrailers;
        let shared = SharedTrailers::new();
        let stream = Box::pin(futures_util::stream::iter(vec![
            Ok(Bytes::from("a")),
            Err(crate::error::Error::Body("boom".into())),
        ]));
        let mut resp = Response::new(
            StatusCode::OK,
            Version::HTTP_11,
            HeaderMap::new(),
            Url::parse("http://example.com").unwrap(),
            ResponseBody::streaming(stream),
        );
        resp.set_trailers(shared);
        let err = resp.bytes().await.unwrap_err();
        assert!(matches!(err, crate::error::Error::Body(_)));
        assert!(
            resp.trailers().is_none(),
            "body errors must not fabricate trailer state"
        );
    }
}