1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
//! Bounded reads of HTTP response bodies.
//!
//! `reqwest`'s `text()`, `json()` and `bytes()` read a body to its end, so a
//! server that sends a huge or endless body (no `Content-Length`, or a false
//! one) is buffered in full and bounded only by the request timeout.
//! [`read_capped`] reads chunk by chunk and stops at a cap, leaving the rest
//! unread; dropping the response then closes the connection. One helper, so
//! the model catalog, the web tools and the remote MCP connectors all read
//! the same way.
/// The most of a body read from a remote server a caller does not control
/// when nothing smaller applies: 8 MiB. The model catalog's download (its
/// backstop ahead of signature verification) and the Parslee relay's
/// largest reply (`car_parslee::relay`'s `MAX_RESPONSE_BYTES`) are held to
/// the same 8 MiB.
pub const MAX_REMOTE_BODY_BYTES: usize = 8 * 1024 * 1024;
/// The start of a response body, as [`read_capped`] read it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CappedBody {
/// The bytes read: the whole body, or its first `cap` bytes when
/// [`cut`](Self::cut) is set.
pub bytes: Vec<u8>,
/// Whether the body went on past the cap. The rest was never read.
pub cut: bool,
/// The response's `Content-Type`, which [`text`](Self::text) decodes by.
pub content_type: Option<String>,
}
impl CappedBody {
/// The bytes decoded by the `Content-Type` charset, the way reqwest's
/// `text()` decodes: UTF-8 when it names none or one encoding_rs does
/// not know, a byte-order mark winning over either, invalid sequences
/// replaced.
pub fn text(&self) -> String {
let encoding = self
.content_type
.as_deref()
.and_then(charset)
.and_then(|label| encoding_rs::Encoding::for_label(label.as_bytes()))
.unwrap_or(encoding_rs::UTF_8);
encoding.decode(&self.bytes).0.into_owned()
}
}
/// Read at most `cap` bytes of `response`'s body. The request's timeout
/// still bounds the read in time.
pub async fn read_capped(
mut response: reqwest::Response,
cap: usize,
) -> reqwest::Result<CappedBody> {
let content_type = response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.map(str::to_string);
let mut bytes = Vec::new();
let mut cut = false;
while let Some(chunk) = response.chunk().await? {
let room = cap - bytes.len();
if chunk.len() > room {
bytes.extend_from_slice(&chunk[..room]);
cut = true;
break;
}
bytes.extend_from_slice(&chunk);
}
Ok(CappedBody {
bytes,
cut,
content_type,
})
}
/// The `charset` parameter of a `Content-Type` value, unquoted.
fn charset(content_type: &str) -> Option<&str> {
content_type.split(';').skip(1).find_map(|parameter| {
let (name, value) = parameter.split_once('=')?;
name.trim()
.eq_ignore_ascii_case("charset")
.then(|| value.trim().trim_matches('"'))
})
}
#[cfg(test)]
mod tests {
use super::*;
fn body(bytes: &[u8], content_type: Option<&str>) -> CappedBody {
CappedBody {
bytes: bytes.to_vec(),
cut: false,
content_type: content_type.map(str::to_string),
}
}
#[test]
fn text_decodes_by_the_named_charset() {
let latin1 = body(b"caf\xe9", Some("text/html; charset=\"ISO-8859-1\""));
assert_eq!(latin1.text(), "café");
let utf16 = body(b"h\0i\0", Some("text/plain;CHARSET=utf-16le"));
assert_eq!(utf16.text(), "hi");
}
#[test]
fn text_falls_back_to_utf8() {
assert_eq!(body("café".as_bytes(), None).text(), "café");
assert_eq!(
body("café".as_bytes(), Some("text/html; charset=no-such")).text(),
"café"
);
assert_eq!(body(b"caf\xe9", Some("text/html")).text(), "caf\u{fffd}");
}
}