from __future__ import annotations
import os
import select
import socket
import socketserver
import threading
from contextlib import contextmanager
from unittest import mock
import httpx
import pytest
import eggfetch
from eggfetch.compat.httpx import Client as CompatClient
from .native_fixtures import local_http_server, local_proxy_server
NO_PROXY_CASES = [
("localhost", "127.0.0.1", True, True),
("localhost", "localhost", True, False),
("127.0.0.1", "127.0.0.1", True, False),
("*", "127.0.0.1", True, False),
(".localhost", "127.0.0.1", False, True),
]
def _request(runtime: str, target: str, *, trust_env: bool = True):
if runtime == "reference":
with httpx.Client(trust_env=trust_env, timeout=3) as client:
return client.get(target)
if runtime == "facade":
with CompatClient(trust_env=trust_env, timeout=3) as client:
return client.get(target)
if runtime == "native_httpx":
with eggfetch.Client(trust_env=trust_env, timeout=3, proxy_env="httpx") as client:
return client.get(target)
with eggfetch.Client(trust_env=trust_env, timeout=3) as client:
return client.get(target)
@pytest.mark.parametrize(
("no_proxy", "target_host", "native_bypasses", "httpx_proxies"),
NO_PROXY_CASES,
)
def test_native_and_compat_use_different_no_proxy_rules(
no_proxy, target_host, native_bypasses, httpx_proxies
):
with local_http_server() as (backend_host, backend_port):
with local_proxy_server(backend=(backend_host, backend_port)) as (
proxy_host,
proxy_port,
handler,
):
env = {
"HTTP_PROXY": f"http://{proxy_host}:{proxy_port}",
"NO_PROXY": no_proxy,
}
target = f"http://{target_host}:{backend_port}/health"
observed: dict[str, bool] = {}
with mock.patch.dict(os.environ, env, clear=True):
for runtime in ("native", "facade", "reference", "native_httpx"):
assert _request(runtime, target).status_code == 200
observed[runtime] = bool(handler.recorded_requests)
handler.recorded_requests.clear()
assert observed["native"] is (not native_bypasses)
assert observed["facade"] is httpx_proxies
assert observed["native_httpx"] is httpx_proxies
assert observed["facade"] is observed["reference"]
def test_native_default_is_native_semantics_not_compat_semantics():
with local_http_server() as (backend_host, backend_port):
with local_proxy_server(backend=(backend_host, backend_port)) as (
proxy_host,
proxy_port,
handler,
):
env = {
"HTTP_PROXY": f"http://{proxy_host}:{proxy_port}",
"NO_PROXY": "localhost",
}
target = f"http://127.0.0.1:{backend_port}/health"
with mock.patch.dict(os.environ, env, clear=True):
assert _request("native", target).status_code == 200
assert handler.recorded_requests == [], (
"the native client must bypass the proxy for a loopback target "
"under eggfetch's own NO_PROXY rules"
)
def test_native_proxy_env_option_opts_into_httpx_semantics_explicitly():
with local_http_server() as (backend_host, backend_port):
with local_proxy_server(backend=(backend_host, backend_port)) as (
proxy_host,
proxy_port,
handler,
):
env = {
"HTTP_PROXY": f"http://{proxy_host}:{proxy_port}",
"NO_PROXY": "localhost",
}
target = f"http://127.0.0.1:{backend_port}/health"
with mock.patch.dict(os.environ, env, clear=True):
with eggfetch.Client(trust_env=True, timeout=3, proxy_env="httpx") as client:
assert client.get(target).status_code == 200
assert handler.recorded_requests, (
"proxy_env='httpx' must apply HTTPX's NO_PROXY rules"
)
handler.recorded_requests.clear()
with eggfetch.Client(trust_env=True, timeout=3, proxy_env="native") as client:
assert client.get(target).status_code == 200
assert handler.recorded_requests == [], (
"proxy_env='native' must apply eggfetch's own NO_PROXY rules"
)
def test_unknown_proxy_env_value_is_rejected():
with pytest.raises(ValueError):
eggfetch.Client(proxy_env="compat-ish")
with pytest.raises(ValueError):
eggfetch.AsyncClient(proxy_env="compat-ish")
def _read_exact(sock: socket.socket, length: int) -> bytes:
data = bytearray()
while len(data) < length:
chunk = sock.recv(length - len(data))
if not chunk:
raise ConnectionError("SOCKS fixture connection closed")
data.extend(chunk)
return bytes(data)
class _SocksHandler(socketserver.BaseRequestHandler):
def handle(self) -> None:
record: dict[str, object] = {}
self.server.records.append(record) client = self.request
client.settimeout(4)
try:
version, method_count = _read_exact(client, 2)
methods = _read_exact(client, method_count)
assert version == 5
record["methods"] = methods
client.sendall(b"\x05\x00")
header = _read_exact(client, 4)
if header[:3] != b"\x05\x01\x00":
return
address_type = header[3]
if address_type == 1:
host = socket.inet_ntoa(_read_exact(client, 4))
elif address_type == 3:
host_length = _read_exact(client, 1)[0]
host = _read_exact(client, host_length).decode("ascii")
elif address_type == 4:
host = socket.inet_ntop(socket.AF_INET6, _read_exact(client, 16))
else:
return
port = int.from_bytes(_read_exact(client, 2), "big")
record["atyp"] = address_type
record["host"] = host
record["port"] = port
client.sendall(b"\x05\x00\x00\x01\x7f\x00\x00\x01\x00\x00")
upstream = socket.create_connection(self.server.origin, timeout=2) with upstream:
client.settimeout(None)
upstream.settimeout(None)
while True:
readable, _, _ = select.select([client, upstream], [], [], 4)
if not readable:
return
for source in readable:
data = source.recv(65536)
if not data:
return
destination = upstream if source is client else client
destination.sendall(data)
except (AssertionError, ConnectionError, OSError, socket.timeout):
return
class _SocksServer(socketserver.ThreadingTCPServer):
allow_reuse_address = True
daemon_threads = True
@contextmanager
def _socks_server(origin: tuple[str, int]):
server = _SocksServer(("127.0.0.1", 0), _SocksHandler)
server.records = [] server.origin = origin thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
yield server
finally:
server.shutdown()
server.server_close()
thread.join(timeout=2)
def _socks_request(runtime: str, proxy_url: str, target: str) -> None:
if runtime == "reference":
with httpx.Client(proxy=proxy_url, trust_env=False, timeout=3) as client:
client.get(target)
return
if runtime == "facade":
with CompatClient(proxy=proxy_url, trust_env=False, timeout=3) as client:
client.get(target)
return
with eggfetch.Client(proxy=proxy_url, trust_env=False, timeout=3) as client:
client.get(target)
def test_native_leaves_socks5_scheme_alone_and_compat_rewrites_it():
with local_http_server() as (backend_host, backend_port):
with _socks_server((backend_host, backend_port)) as server:
proxy_url = f"socks5://127.0.0.1:{server.server_address[1]}"
target = f"http://localhost:{backend_port}/health"
server.records.clear()
_socks_request("native", proxy_url, target)
native_records = list(server.records)
server.records.clear()
_socks_request("facade", proxy_url, target)
facade_records = list(server.records)
server.records.clear()
_socks_request("reference", proxy_url, target)
reference_records = list(server.records)
assert native_records, "the native client did not reach the SOCKS proxy"
assert facade_records, "the compat facade did not reach the SOCKS proxy"
assert native_records[0]["atyp"] in (1, 4), native_records
assert native_records[0]["host"] != "localhost", native_records
assert facade_records[0]["atyp"] == 3, facade_records
assert facade_records[0]["host"] == "localhost", facade_records
assert reference_records, "upstream httpx did not reach the SOCKS proxy"
assert reference_records[0]["atyp"] == 3, reference_records
assert reference_records[0]["host"] == "localhost", reference_records
def test_native_proxy_env_option_selects_the_socks_scheme_policy():
with local_http_server() as (backend_host, backend_port):
with _socks_server((backend_host, backend_port)) as server:
proxy_url = f"socks5://127.0.0.1:{server.server_address[1]}"
target = f"http://localhost:{backend_port}/health"
server.records.clear()
with eggfetch.Client(
proxy=proxy_url, trust_env=False, timeout=3, proxy_env="httpx"
) as client:
client.get(target)
httpx_records = list(server.records)
assert httpx_records, "the native client did not reach the SOCKS proxy"
assert httpx_records[0]["atyp"] == 3, httpx_records
assert httpx_records[0]["host"] == "localhost", httpx_records
def test_httpx2_client_inherits_the_compat_environment_semantics():
from eggfetch.compat import httpx2
with local_http_server() as (backend_host, backend_port):
with local_proxy_server(backend=(backend_host, backend_port)) as (
proxy_host,
proxy_port,
handler,
):
env = {
"HTTP_PROXY": f"http://{proxy_host}:{proxy_port}",
"NO_PROXY": "localhost",
}
target = f"http://127.0.0.1:{backend_port}/health"
with mock.patch.dict(os.environ, env, clear=True):
with httpx2.Client(trust_env=True, timeout=3) as client:
assert client.get(target).status_code == 200
assert handler.recorded_requests, (
"httpx2 must apply HTTPX's NO_PROXY rules through the "
"shared base facade"
)
def test_httpx2_does_not_redefine_client_construction():
from eggfetch.compat import httpx2
from eggfetch.compat.httpx import _client as base_client
assert httpx2.Client.__init__ is base_client.Client.__init__
assert httpx2.AsyncClient.__init__ is base_client.AsyncClient.__init__
assert "_ensure_client" not in httpx2.Client.__dict__
assert "_ensure_client" not in httpx2.AsyncClient.__dict__