use std::sync::{Arc, OnceLock};
use bytes::Bytes;
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyDict, PyModule, PyString};
use crate::cookies::PyCookies;
use crate::errors::HTTPStatusError;
use crate::headers::PyHeaders;
use crate::network_stream::{EitherNetworkStream, PyAsyncNetworkStream, PyNetworkStream};
use crate::streaming::{safe_url_for_display, RuntimeLease};
static NOOP_ACLOSE: OnceLock<Py<PyAny>> = OnceLock::new();
fn noop_aclose_coroutine(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
if let Some(func) = NOOP_ACLOSE.get() {
return Ok(func.bind(py).clone());
}
let module = PyModule::from_code(
py,
c"async def _noop_aclose():\n pass\n",
c"<eggfetch._response>",
c"eggfetch._response",
)?;
let func: Bound<'_, PyAny> = module.getattr("_noop_aclose")?;
let _ = NOOP_ACLOSE.set(func.clone().unbind());
Ok(func)
}
pub(crate) fn version_to_string(version: http::Version) -> String {
match version {
http::Version::HTTP_10 => "HTTP/1.0".to_string(),
http::Version::HTTP_11 => "HTTP/1.1".to_string(),
http::Version::HTTP_2 => "HTTP/2".to_string(),
http::Version::HTTP_3 => "HTTP/3".to_string(),
other => format!("{other:?}"),
}
}
pub(crate) fn extract_charset(headers: &http::HeaderMap) -> Option<String> {
let content_type = headers.get("content-type")?;
let ct_str = content_type.to_str().ok()?;
for part in ct_str.split(';').skip(1) {
let part = part.trim();
if let Some((name, charset)) = part.split_once('=') {
if name.trim().eq_ignore_ascii_case("charset") {
let charset = charset.trim().trim_matches(['"', '\'']);
return Some(charset.to_string());
}
}
}
None
}
fn decode_with_encoding(content: &[u8], encoding: Option<&str>) -> String {
if let Some(enc_name) = encoding {
if let Some(enc) = encoding_rs::Encoding::for_label(enc_name.as_bytes()) {
let (decoded, _, _) = enc.decode(content);
return decoded.into_owned();
}
}
String::from_utf8_lossy(content).to_string()
}
#[pyclass(name = "Response", from_py_object)]
#[derive(Clone)]
pub struct PyResponse {
#[pyo3(get)]
status_code: u16,
#[pyo3(get)]
headers: PyHeaders,
#[pyo3(get)]
url: String,
content: Bytes,
text: OnceLock<Arc<str>>,
#[pyo3(get)]
reason_phrase: String,
#[pyo3(get)]
http_version: String,
#[pyo3(get)]
encoding: Option<String>,
#[pyo3(get)]
history: Vec<PyResponse>,
#[pyo3(get)]
cookies: PyCookies,
#[pyo3(get)]
_stream_consumed: bool,
#[pyo3(get)]
_wire_content_encoding: Option<String>,
#[pyo3(get)]
_wire_content_length: Option<String>,
_network_stream: Option<EitherNetworkStream>,
}
impl std::fmt::Debug for PyResponse {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PyResponse")
.field("status_code", &self.status_code)
.field("headers", &self.headers)
.field("url", &safe_url_for_display(&self.url))
.field("content", &format!("<{} bytes>", self.content.len()))
.field("cookies", &self.cookies)
.finish_non_exhaustive()
}
}
impl PyResponse {
#[allow(clippy::unnecessary_wraps, clippy::needless_pass_by_value)]
pub(crate) fn from_core_response_with_body(
response: &mut eggfetch_core::Response,
content: Bytes,
runtime_handle: Option<&tokio::runtime::Handle>,
runtime_lease: Option<&RuntimeLease>,
is_async: bool,
) -> PyResult<Self> {
let status = response.status().as_u16();
let wire_content_encoding = response.wire_content_encoding().map(ToOwned::to_owned);
let wire_content_length = response.wire_content_length().map(ToOwned::to_owned);
let reason_phrase = response
.wire_reason_phrase()
.map(ToOwned::to_owned)
.or_else(|| response.status().canonical_reason().map(ToOwned::to_owned))
.unwrap_or_default();
let http_version = version_to_string(response.version());
let encoding = extract_charset(response.headers());
let response_url = response.url().to_string();
let set_cookie_headers: Vec<String> = response
.headers()
.get_all("set-cookie")
.iter()
.filter_map(|v| v.to_str().ok().map(ToString::to_string))
.collect();
let core_history = std::mem::take(response.history_mut());
let history: Vec<PyResponse> = core_history
.into_iter()
.map(|entry| {
let status = entry.status().as_u16();
let headers = PyHeaders::from_header_map(entry.headers().clone());
let url = entry.url().to_string();
let reason_phrase = entry.reason_phrase().to_string();
let http_version = version_to_string(entry.version());
let encoding = extract_charset(entry.headers());
PyResponse::from_parts(
status,
headers,
url,
Bytes::new(),
reason_phrase,
http_version,
encoding,
)
})
.collect();
let jar = eggfetch_core::cookie::CookieJar::new();
if !set_cookie_headers.is_empty() {
jar.update_from_response(response.url(), &set_cookie_headers);
}
let cookies = PyCookies::from_jar(jar);
let headers = PyHeaders::from_header_map(std::mem::take(response.headers_mut()));
let network_stream = response
.take_network_stream()
.map(|ns| -> PyResult<_> {
match ns {
eggfetch_core::network_stream::NetworkStream::Upgraded(u) => {
if is_async {
Ok(EitherNetworkStream::Async(
PyAsyncNetworkStream::from_upgraded(u),
))
} else if let Some(handle) = runtime_handle {
Ok(EitherNetworkStream::Sync(
PyNetworkStream::from_upgraded_with_handle(
u,
handle.clone(),
runtime_lease.cloned(),
),
))
} else {
Err(PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(
"received a 101 upgrade response but no runtime handle is \
available to drive upgraded-stream IO; use a Client instance \
instead of the one-shot helpers for upgrade responses",
))
}
}
eggfetch_core::network_stream::NetworkStream::Metadata(m) => {
if is_async {
Ok(EitherNetworkStream::Async(
PyAsyncNetworkStream::from_metadata(m)?,
))
} else {
Ok(EitherNetworkStream::Sync(PyNetworkStream::from_metadata(
m,
)?))
}
}
}
})
.transpose()?;
Ok(Self {
status_code: status,
headers,
url: response_url,
content,
text: OnceLock::new(),
reason_phrase,
http_version,
encoding,
history,
cookies,
_stream_consumed: false,
_wire_content_encoding: wire_content_encoding,
_wire_content_length: wire_content_length,
_network_stream: network_stream,
})
}
pub fn from_parts(
status: u16,
headers: PyHeaders,
url: String,
content: Bytes,
reason_phrase: String,
http_version: String,
encoding: Option<String>,
) -> Self {
Self {
status_code: status,
headers,
url,
content,
text: OnceLock::new(),
reason_phrase,
http_version,
encoding,
history: Vec::new(),
cookies: PyCookies::from_jar(eggfetch_core::cookie::CookieJar::new()),
_stream_consumed: false,
_wire_content_encoding: None,
_wire_content_length: None,
_network_stream: None,
}
}
fn raise_for_status_message(&self) -> String {
format!(
"{} {} for url '{}'",
self.status_code,
self.reason_phrase,
safe_url_for_display(&self.url)
)
}
fn decoded_text(&self) -> &str {
self.text
.get_or_init(|| {
Arc::<str>::from(decode_with_encoding(
&self.content,
self.encoding.as_deref(),
))
})
.as_ref()
}
fn decoded_text_arc(&self) -> Arc<str> {
Arc::clone(self.text.get_or_init(|| {
Arc::<str>::from(decode_with_encoding(
&self.content,
self.encoding.as_deref(),
))
}))
}
}
#[pyclass(name = "_ResponseBytesIterator")]
pub(crate) struct PyResponseBytesIterator {
response: Option<Py<PyResponse>>,
content: Option<Bytes>,
chunk_size: usize,
cursor: usize,
}
#[pymethods]
impl PyResponseBytesIterator {
fn __iter__(slf: Py<Self>) -> Py<Self> {
slf
}
fn __next__(&mut self, py: Python<'_>) -> Option<Py<PyAny>> {
if self.content.is_none() {
let content = self
.response
.as_ref()
.map(|response| response.bind(py).borrow().content.clone())?;
self.content = Some(content);
self.response = None;
}
let content = self.content.as_ref()?;
if self.cursor >= content.len() {
return None;
}
let end = self
.cursor
.saturating_add(self.chunk_size)
.min(content.len());
let chunk = PyBytes::new(py, &content[self.cursor..end]).into();
self.cursor = end;
Some(chunk)
}
}
#[pyclass(name = "_ResponseTextIterator")]
pub(crate) struct PyResponseTextIterator {
response: Option<Py<PyResponse>>,
text: Option<Arc<str>>,
chunk_size: usize,
byte_cursor: usize,
}
#[pymethods]
impl PyResponseTextIterator {
fn __iter__(slf: Py<Self>) -> Py<Self> {
slf
}
fn __next__(&mut self, py: Python<'_>) -> Option<Py<PyAny>> {
if self.text.is_none() {
let text = self
.response
.as_ref()
.map(|response| response.bind(py).borrow().decoded_text_arc())?;
self.text = Some(text);
self.response = None;
}
let text = self.text.as_ref()?;
if self.byte_cursor >= text.len() {
return None;
}
let start = self.byte_cursor;
let mut chars = text[start..].char_indices();
let mut count = 0;
let end = loop {
match chars.next() {
Some((offset, character)) => {
count += 1;
if count == self.chunk_size {
break start + offset + character.len_utf8();
}
}
None => break text.len(),
}
};
let value = PyString::new(py, &text[start..end]).into();
self.byte_cursor = end;
Some(value)
}
}
#[pyclass(name = "_ResponseLinesIterator")]
pub(crate) struct PyResponseLinesIterator {
response: Option<Py<PyResponse>>,
text: Option<Arc<str>>,
byte_cursor: usize,
}
#[pymethods]
impl PyResponseLinesIterator {
fn __iter__(slf: Py<Self>) -> Py<Self> {
slf
}
fn __next__(&mut self, py: Python<'_>) -> Option<Py<PyAny>> {
if self.text.is_none() {
let text = self
.response
.as_ref()
.map(|response| response.bind(py).borrow().decoded_text_arc())?;
self.text = Some(text);
self.response = None;
}
let text = self.text.as_ref()?;
if self.byte_cursor >= text.len() {
return None;
}
let start = self.byte_cursor;
let (mut end, next_cursor, terminated_by_lf) = match text[start..].find('\n') {
Some(offset) => (start + offset, start + offset + 1, true),
None => (text.len(), text.len(), false),
};
if terminated_by_lf && end > start && text.as_bytes()[end - 1] == b'\r' {
end -= 1;
}
self.byte_cursor = next_cursor;
Some(PyString::new(py, &text[start..end]).into())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn raise_for_status_message_redacts_url_credentials() {
let resp = PyResponse::from_parts(
404,
PyHeaders::from_header_map(http::HeaderMap::new()),
"https://user:secret@host.example/path?q=hush".to_owned(),
Bytes::new(),
"Not Found".to_owned(),
"HTTP/1.1".to_owned(),
None,
);
let msg = resp.raise_for_status_message();
assert!(msg.contains("host.example"));
assert!(!msg.contains("secret"), "leaked password: {msg}");
assert!(!msg.contains("user:"), "leaked username: {msg}");
assert!(!msg.contains("q=hush"), "leaked query: {msg}");
}
}
#[pymethods]
impl PyResponse {
fn raise_for_status(&self) -> PyResult<()> {
if self.status_code >= 400 {
return Err(HTTPStatusError::new_err(self.raise_for_status_message()));
}
Ok(())
}
#[getter]
fn is_informational(&self) -> bool {
(100..200).contains(&self.status_code)
}
#[getter]
fn is_success(&self) -> bool {
(200..300).contains(&self.status_code)
}
#[getter]
fn is_redirect(&self) -> bool {
(300..400).contains(&self.status_code)
}
#[getter]
fn is_client_error(&self) -> bool {
(400..500).contains(&self.status_code)
}
#[getter]
fn is_server_error(&self) -> bool {
(500..600).contains(&self.status_code)
}
#[getter]
fn is_error(&self) -> bool {
(400..600).contains(&self.status_code)
}
#[getter]
fn content<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
PyBytes::new(py, &self.content)
}
#[getter]
fn text(&self) -> &str {
self.decoded_text()
}
#[pyo3(signature = (**kwargs))]
fn json(&self, py: Python<'_>, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<Py<PyAny>> {
let json_module = py.import("json")?;
let text_obj = PyString::new(py, self.decoded_text());
let loads = json_module.getattr("loads")?;
match kwargs {
Some(kw) => loads.call((text_obj,), Some(kw)).map(Into::into),
None => loads.call1((text_obj,)).map(Into::into),
}
}
#[pyo3(signature = (chunk_size=8192))]
fn iter_bytes(
slf: Py<Self>,
py: Python<'_>,
chunk_size: usize,
) -> PyResult<Bound<'_, PyResponseBytesIterator>> {
if chunk_size == 0 {
return Err(pyo3::exceptions::PyValueError::new_err(
"chunk_size must be greater than zero",
));
}
Py::new(
py,
PyResponseBytesIterator {
response: Some(slf),
content: None,
chunk_size,
cursor: 0,
},
)
.map(|iterator| iterator.into_bound(py))
}
#[pyo3(signature = (chunk_size=8192))]
fn iter_text(
slf: Py<Self>,
py: Python<'_>,
chunk_size: usize,
) -> PyResult<Bound<'_, PyResponseTextIterator>> {
if chunk_size == 0 {
return Err(pyo3::exceptions::PyValueError::new_err(
"chunk_size must be greater than zero",
));
}
Py::new(
py,
PyResponseTextIterator {
response: Some(slf),
text: None,
chunk_size,
byte_cursor: 0,
},
)
.map(|iterator| iterator.into_bound(py))
}
fn iter_lines(slf: Py<Self>, py: Python<'_>) -> PyResult<Bound<'_, PyResponseLinesIterator>> {
Py::new(
py,
PyResponseLinesIterator {
response: Some(slf),
text: None,
byte_cursor: 0,
},
)
.map(|iterator| iterator.into_bound(py))
}
#[allow(clippy::unused_self)] fn close(&self) {}
#[allow(clippy::unused_self)] fn aclose<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let func = noop_aclose_coroutine(py)?;
func.call0()
}
#[getter]
fn extensions<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
let dict = PyDict::new(py);
dict.set_item("http_version", self.http_version.clone())?;
dict.set_item("reason_phrase", self.reason_phrase.clone())?;
#[allow(clippy::used_underscore_binding)]
match self._network_stream.as_ref() {
Some(stream) if stream.is_upgraded() => {
stream.insert_into_dict(py, &dict)?;
}
_ => {
dict.set_item("network_stream", py.None())?;
}
}
Ok(dict)
}
fn __repr__(&self) -> String {
format!("<Response [{} {}]>", self.status_code, self.reason_phrase)
}
}
#[allow(dead_code)] pub(crate) fn response_extensions_from_core<'py>(
py: Python<'py>,
response: &eggfetch_core::Response,
network_stream: Option<&EitherNetworkStream>,
) -> PyResult<Bound<'py, PyDict>> {
let dict = PyDict::new(py);
dict.set_item("http_version", version_to_string(response.version()))?;
let reason = response
.wire_reason_phrase()
.map(ToOwned::to_owned)
.or_else(|| response.status().canonical_reason().map(ToOwned::to_owned))
.unwrap_or_default();
dict.set_item("reason_phrase", reason)?;
match network_stream {
Some(stream) if stream.is_upgraded() => {
stream.insert_into_dict(py, &dict)?;
}
_ => {
dict.set_item("network_stream", py.None())?;
}
}
Ok(dict)
}