use std::collections::{HashMap, HashSet, VecDeque};
use std::net::SocketAddr;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use tokio::net::TcpStream;
use tokio::time::timeout;
use url::Url;
use aria2_protocol::http::response::is_redirect_status;
use crate::error::{Aria2Error, RecoverableError, Result};
use crate::http::cookie_storage::{CookieJar, CookieStorage};
use super::active_connection::{ActiveConnection, ConnectionPoolKey, ProxyInfo};
use super::types::{HttpConfig, HttpResponse};
use crate::network::ConnectionContext;
pub struct HttpConnectionManager {
config: HttpConfig,
pool: HashMap<u64, ActiveConnection>,
key_connections: HashMap<ConnectionPoolKey, VecDeque<u64>>,
active_count: usize,
id_counter: AtomicU64,
max_redirects: u32,
cookie_storage: Option<std::sync::Arc<CookieStorage>>,
cookie_jar: Option<CookieJar>,
}
impl HttpConnectionManager {
pub fn new(config: &HttpConfig) -> Self {
Self {
config: config.clone(),
pool: HashMap::new(),
key_connections: HashMap::new(),
active_count: 0,
id_counter: AtomicU64::new(1),
max_redirects: crate::constants::HTTP_DEFAULT_MAX_REDIRECTS as u32,
cookie_storage: None,
cookie_jar: None,
}
}
pub fn max_connections(&self) -> usize {
self.config.max_connections
}
pub fn active_count(&self) -> usize {
self.active_count
}
pub fn pool_size(&self) -> usize {
self.pool.len()
}
pub fn idle_count_for_key(&self, key: &ConnectionPoolKey) -> usize {
self.key_connections.get(key).map_or(0, |dq| dq.len())
}
pub fn max_idle_per_host(&self) -> usize {
self.config.max_idle_per_host
}
pub async fn acquire(
&mut self,
url: &Url,
proxy: Option<&ProxyInfo>,
) -> Result<ActiveConnection> {
let host = Self::extract_host(url);
let pool_key = ConnectionPoolKey {
target: format!(
"{scheme}://{host}:{port}",
scheme = url.scheme(),
host = host,
port = url.port_or_known_default().unwrap_or(80)
),
proxy: proxy.cloned(),
};
if let Some(conn) = self.pop_pooled_connection(&pool_key)? {
tracing::debug!("Reused connection: id={}, key={:?}", conn.id, pool_key);
return Ok(conn);
}
if self.active_count >= self.config.max_connections {
self.check_timeout();
if self.active_count >= self.config.max_connections {
return Err(Aria2Error::Recoverable(
RecoverableError::TemporaryNetworkFailure {
message: format!(
"Max connection limit reached: {} (key={:?})",
self.config.max_connections, pool_key
),
},
));
}
}
self.create_new_connection(url, &host, pool_key).await
}
pub async fn put_back(&mut self, mut conn: ActiveConnection) {
if !conn.is_valid() {
tracing::debug!("Connection no longer valid on put_back: id={}", conn.id);
self.active_count = self.active_count.saturating_sub(1);
return;
}
let conn_id = conn.id;
let pool_key = conn.pool_key.clone();
conn.mark_pooled();
self.pool.insert(conn_id, conn);
self.key_connections
.entry(pool_key.clone())
.or_default()
.push_back(conn_id);
self.enforce_idle_limit(&pool_key).await;
tracing::debug!(
"Put back connection to pool: id={}, key={:?}",
conn_id,
pool_key
);
}
pub async fn release(&mut self, conn: ActiveConnection) {
self.put_back(conn).await
}
pub async fn discard(&mut self, mut conn: ActiveConnection) {
self.active_count = self.active_count.saturating_sub(1);
let _ = conn.shutdown().await;
}
pub async fn evict_peer(&mut self, context: &ConnectionContext) -> usize {
let ids: Vec<u64> = self
.pool
.iter()
.filter_map(|(id, conn)| {
(!conn.is_proxied() && conn.connection_context() == context).then_some(*id)
})
.collect();
let mut evicted = 0;
for id in ids {
if let Some(mut conn) = self.pool.remove(&id) {
self.remove_from_key_deque(&conn.pool_key, id);
let _ = conn.shutdown().await;
self.active_count = self.active_count.saturating_sub(1);
evicted += 1;
}
}
evicted
}
pub fn get_connection(
&mut self,
pool_key: &ConnectionPoolKey,
) -> Result<Option<ActiveConnection>> {
self.pop_pooled_connection(pool_key)
}
pub fn check_timeout(&mut self) -> usize {
if self.pool.is_empty() {
return 0;
}
let idle_timeout = self.config.idle_timeout;
let mut evicted_keys: Vec<(u64, ConnectionPoolKey)> = Vec::new();
for (&conn_id, conn) in &self.pool {
if conn.is_idle_timeout(idle_timeout) {
evicted_keys.push((conn_id, conn.pool_key.clone()));
}
}
let count = evicted_keys.len();
for (conn_id, pool_key) in evicted_keys {
if let Some(mut conn) = self.pool.remove(&conn_id) {
tracing::debug!(
"Evicted idle-timeout connection: id={}, idle={:.2}s",
conn_id,
conn.pooled_at.map_or(0.0, |t| t.elapsed().as_secs_f64())
);
std::mem::drop(conn.shutdown());
self.active_count = self.active_count.saturating_sub(1);
self.remove_from_key_deque(&pool_key, conn_id);
}
}
if count > 0 {
tracing::info!("Evicted {} idle-timeout connections", count);
}
count
}
pub async fn release_all(&mut self) {
let count = self.pool.len();
for (_, mut conn) in self.pool.drain() {
let _ = conn.shutdown().await;
}
self.key_connections.clear();
self.active_count = self.active_count.saturating_sub(count);
if count > 0 {
tracing::info!("Released all {} idle connections", count);
}
}
pub async fn cleanup(&mut self) {
for (_, mut conn) in self.pool.drain() {
let _ = conn.shutdown().await;
}
self.key_connections.clear();
self.active_count = 0;
tracing::info!("Connection pool cleaned up");
}
pub async fn close_connection(&mut self, conn_id: u64) {
if let Some(mut conn) = self.pool.remove(&conn_id) {
let pool_key = conn.pool_key.clone();
let _ = conn.shutdown().await;
self.active_count = self.active_count.saturating_sub(1);
self.remove_from_key_deque(&pool_key, conn_id);
tracing::debug!("Force closed connection: id={}", conn_id);
}
}
pub fn follow_redirects(
&self,
response: &HttpResponse,
current_url: &Url,
redirect_chain: &HashSet<Url>,
redirect_count: u32,
) -> Result<Url> {
if !is_redirect_status(response.status_code) {
return Err(Aria2Error::Parse(format!(
"Non-redirect response code: {}",
response.status_code
)));
}
if redirect_count >= self.max_redirects {
return Err(Aria2Error::Network(format!(
"Max redirect count exceeded: {}",
self.max_redirects
)));
}
let location = response
.location()
.ok_or_else(|| Aria2Error::Parse("Missing Location header".to_string()))?;
let new_url = current_url
.join(location)
.map_err(|e| Aria2Error::Parse(format!("Failed to parse redirect URL: {}", e)))?;
if redirect_chain.contains(&new_url) {
return Err(Aria2Error::Network(format!(
"Circular redirect detected: {}",
new_url
)));
}
tracing::info!(
"Following redirect: {} -> {} ({}/{})",
current_url,
new_url,
redirect_count + 1,
self.max_redirects
);
Ok(new_url)
}
pub async fn follow_redirects_iterative<F, Fut>(
&self,
initial_url: &Url,
mut get_response: F,
) -> Result<HttpResponse>
where
F: FnMut(&Url) -> Fut,
Fut: std::future::Future<Output = Result<HttpResponse>>,
{
const MAX_REDIRECTS: u8 = crate::constants::HTTP_DEFAULT_MAX_REDIRECTS as u8;
let mut current_url = initial_url.clone();
let mut seen_urls = HashSet::<String>::new();
for iteration in 0..MAX_REDIRECTS {
let url_str = current_url.to_string();
if !seen_urls.insert(url_str.clone()) {
return Err(Aria2Error::Network(format!(
"Redirect loop detected: {}",
url_str
)));
}
let resp = get_response(¤t_url).await?;
if !is_redirect_status(resp.status_code) {
return Ok(resp);
}
let location = resp.location().ok_or_else(|| {
Aria2Error::Network("Missing Location header in redirect response".into())
})?;
current_url = current_url
.join(location)
.map_err(|e| Aria2Error::Parse(format!("Failed to parse redirect URL: {}", e)))?;
tracing::info!(
"Following redirect: iteration {}/{}",
iteration + 1,
MAX_REDIRECTS
);
}
Err(Aria2Error::Network(format!(
"Too many redirects (>{}), last URL: {}",
MAX_REDIRECTS, current_url
)))
}
pub fn build_range_header(&self, start: u64, end: Option<u64>) -> String {
match end {
Some(e) => format!("bytes={}-{}", start, e),
None => format!("bytes={}-", start),
}
}
pub fn parse_content_range(&self, header: &str) -> Option<(u64, u64, u64)> {
let header = header.trim();
let range = header
.strip_prefix("bytes ")
.or_else(|| header.strip_prefix("bytes="))
.unwrap_or(header);
let (range, total) = range.split_once('/')?;
let (start, end) = range.split_once('-')?;
let start = start.trim().parse::<u64>().ok()?;
let end = end.trim().parse::<u64>().ok()?;
if start > end {
return None;
}
let total = match total.trim() {
"*" => u64::MAX,
value => value.parse().ok()?,
};
if total != u64::MAX && end >= total {
return None;
}
Some((start, end, total))
}
pub fn set_cookie_storage(&mut self, storage: Option<std::sync::Arc<CookieStorage>>) {
self.cookie_storage = storage;
}
pub fn with_shared_cookie_storage(config: &HttpConfig) -> Self {
let mut manager = Self::new(config);
manager.set_cookie_storage(Some(CookieStorage::shared()));
manager
}
pub fn set_cookie_jar(&mut self, jar: Option<CookieJar>) {
self.cookie_jar = jar;
}
pub fn cookie_jar(&self) -> &Option<CookieJar> {
&self.cookie_jar
}
pub fn cookie_jar_mut(&mut self) -> &mut Option<CookieJar> {
&mut self.cookie_jar
}
pub fn attach_cookies_to_request(&self, url: &Url) -> Option<String> {
let is_https = url.scheme() == "https";
if let Some(storage) = &self.cookie_storage {
let header =
storage.to_header_string(url.host_str().unwrap_or_default(), url.path(), is_https);
if !header.is_empty() {
return Some(header);
}
}
let jar = self.cookie_jar.as_ref()?;
jar.cookie_header_for_url(url.as_str(), is_https)
}
pub fn extract_cookies_from_response(
&mut self,
response_headers: &[(String, String)],
_request_url: &Url,
) -> usize {
let mut stored = 0;
if let Some(storage) = &self.cookie_storage {
for (name, value) in response_headers {
if name.eq_ignore_ascii_case("set-cookie")
&& storage.parse_and_store(
value,
_request_url.host_str().unwrap_or_default(),
_request_url.path(),
)
{
stored += 1;
}
}
return stored;
}
let jar = match &mut self.cookie_jar {
Some(j) => j,
None => return 0,
};
for (name, value) in response_headers {
if name.eq_ignore_ascii_case("set-cookie")
&& let Some(cookie) = crate::http::cookie::JarCookie::parse_set_cookie(value)
{
jar.store(cookie);
stored += 1;
tracing::debug!(
"Extracted cookie from Set-Cookie: {}",
&value[..value.len().min(80)]
);
}
}
stored
}
pub(super) fn extract_host(url: &Url) -> String {
match url.port_or_known_default() {
Some(port) => format!(
"{}:{}",
url.host_str().unwrap_or(crate::constants::DEFAULT_HOST),
port
),
None => url
.host_str()
.unwrap_or(crate::constants::DEFAULT_HOST)
.to_string(),
}
}
fn pop_pooled_connection(
&mut self,
pool_key: &ConnectionPoolKey,
) -> Result<Option<ActiveConnection>> {
let deque = match self.key_connections.get_mut(pool_key) {
Some(dq) => dq,
None => return Ok(None),
};
let mut candidate_ids: Vec<u64> = Vec::new();
while let Some(&conn_id) = deque.front() {
candidate_ids.push(conn_id);
deque.pop_front();
if let Some(mut conn) = self.pool.remove(&conn_id) {
if conn.is_valid() && !conn.is_idle_timeout(self.config.idle_timeout) {
conn.mark_in_use();
conn.touch();
tracing::debug!(
"Reused idle connection: id={}, idle={:.2}s",
conn_id,
conn.pooled_at.map_or(0.0, |t| t.elapsed().as_secs_f64())
);
return Ok(Some(conn));
} else {
let reason = if !conn.is_valid() {
"invalid"
} else {
"timed-out"
};
tracing::debug!("Discarded {} idle connection: id={}", reason, conn_id);
std::mem::drop(conn.shutdown());
self.active_count = self.active_count.saturating_sub(1);
}
}
}
if deque.is_empty() {
self.key_connections.remove(pool_key);
}
Ok(None)
}
async fn create_new_connection(
&mut self,
url: &Url,
host: &str,
pool_key: ConnectionPoolKey,
) -> Result<ActiveConnection> {
let addresses = Self::resolve_addresses(url).await?;
let mut last_error = None;
let mut connected = None;
for addr in &addresses {
match timeout(self.config.connect_timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => {
connected = Some((stream, *addr));
break;
}
Ok(Err(error)) => last_error = Some(format!("{}: {}", addr, error)),
Err(_) => last_error = Some(format!("{}: connection timeout", addr)),
}
}
let (stream, _addr) = connected.ok_or_else(|| {
Aria2Error::Recoverable(RecoverableError::TemporaryNetworkFailure {
message: format!(
"TCP connection failed for {}: {}",
host,
last_error.unwrap_or_else(|| "no addresses".to_string())
),
})
})?;
if let Err(e) = stream.set_nodelay(true) {
tracing::warn!("Failed to set nodelay: {}", e);
}
let conn_id = self.id_counter.fetch_add(1, Ordering::SeqCst);
let conn = ActiveConnection {
id: conn_id,
connection: ConnectionContext::new(
host,
url.port_or_known_default().unwrap_or(80),
stream.peer_addr().map_err(|e| {
Aria2Error::Recoverable(RecoverableError::TemporaryNetworkFailure {
message: format!("peer address unavailable: {e}"),
})
})?,
),
stream,
host: host.to_string(),
last_used: Instant::now(),
pooled_at: None, pool_key: pool_key.clone(),
};
self.active_count += 1;
tracing::info!(
"Created new connection: id={}, host={}, active={}/{}",
conn_id,
host,
self.active_count,
self.config.max_connections
);
Ok(conn)
}
async fn resolve_addresses(url: &Url) -> Result<Vec<SocketAddr>> {
let host = url
.host_str()
.ok_or_else(|| Aria2Error::Parse("URL missing hostname".to_string()))?;
let port = url
.port_or_known_default()
.ok_or_else(|| Aria2Error::Parse("Unable to determine port number".to_string()))?;
let host_port = format!("{}:{}", host, port);
tokio::net::lookup_host(&host_port)
.await
.map(|addresses| addresses.collect())
.map_err(|error| Aria2Error::Parse(format!("Failed to resolve address: {}", error)))
}
async fn enforce_idle_limit(&mut self, pool_key: &ConnectionPoolKey) {
let max_idle = self.config.max_idle_per_host;
if max_idle == 0 {
return; }
while self.idle_count_for_key(pool_key) > max_idle {
let evict_id = match self.key_connections.get_mut(pool_key) {
Some(dq) => match dq.pop_front() {
Some(id) => id,
None => break,
},
None => break,
};
if let Some(mut conn) = self.pool.remove(&evict_id) {
tracing::debug!(
"LRU evicted oldest idle connection: id={}, key={:?} (limit={})",
evict_id,
pool_key,
max_idle
);
let _ = conn.shutdown().await;
self.active_count = self.active_count.saturating_sub(1);
}
}
if self.idle_count_for_key(pool_key) == 0 {
self.key_connections.remove(pool_key);
}
}
fn remove_from_key_deque(&mut self, pool_key: &ConnectionPoolKey, conn_id: u64) {
if let Some(dq) = self.key_connections.get_mut(pool_key) {
dq.retain(|&id| id != conn_id);
if dq.is_empty() {
self.key_connections.remove(pool_key);
}
}
}
}
impl Drop for HttpConnectionManager {
fn drop(&mut self) {
for (_, conn) in self.pool.drain() {
drop(conn);
}
self.key_connections.clear();
}
}
impl std::fmt::Debug for HttpConnectionManager {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HttpConnectionManager")
.field("max_connections", &self.config.max_connections)
.field("connect_timeout", &self.config.connect_timeout)
.field("read_timeout", &self.config.read_timeout)
.field("write_timeout", &self.config.write_timeout)
.field("idle_timeout", &self.config.idle_timeout)
.field("max_idle_per_host", &self.config.max_idle_per_host)
.field("active_count", &self.active_count)
.field("pool_size", &self.pool.len())
.field("cookie_storage_set", &self.cookie_storage.is_some())
.field("cookie_jar_set", &self.cookie_jar.is_some())
.finish()
}
}