use std::sync::Arc;
use std::time::Instant;
use tracing::debug;
use super::entry::FileEntry;
use super::helpers::{extract_host, extract_host_and_protocol};
use super::types::{NUM_URI_SCAN, SPEED_THRESHOLD, STARTUP_IDLE_TIME};
use crate::download::request::Request;
use crate::selector::server_stat_man::ServerStatMan;
use crate::selector::uri_selector::UriSelector;
impl FileEntry {
pub fn get_request(
&mut self,
selector: &dyn UriSelector,
uri_reuse: bool,
used_hosts: &[(usize, String)],
referer: &str,
method: &str,
) -> Option<Arc<Request>> {
self.sort_pool_by_speed();
if self.request_pool.is_empty() {
let in_flight_hosts = self.collect_in_flight_hosts();
return self.get_request_with_in_flight_hosts(
selector,
uri_reuse,
used_hosts,
referer,
method,
&in_flight_hosts,
);
}
let now = Instant::now();
let awake_idx = self
.request_pool
.iter()
.position(|req| req.wake_time() <= now);
let req;
if let Some(idx) = awake_idx {
req = Some(self.request_pool.remove(idx));
} else {
let mut in_flight_hosts = self.collect_in_flight_hosts();
for pooled in &self.request_pool {
if let Some(host) = extract_host(pooled.uri()) {
in_flight_hosts.push(host);
}
}
let uri_req = self.get_request_with_in_flight_hosts(
selector,
uri_reuse,
used_hosts,
referer,
method,
&in_flight_hosts,
);
match uri_req {
Some(r) => {
let first_pool_uri = self.request_pool.first().map(|r| r.uri().to_owned());
if first_pool_uri.as_deref() == Some(r.uri()) {
req = Some(self.request_pool.remove(0));
} else {
req = Some(r);
}
}
None => {
req = Some(self.request_pool.remove(0));
}
}
}
if let Some(ref r) = req {
debug!("Picked up from pool: {}", r.uri());
}
if let Some(r) = req {
self.in_flight_requests.push(Arc::clone(&r));
Some(r)
} else {
None
}
}
pub fn pool_request(&mut self, request: &Arc<Request>) {
self.remove_request(request);
if !request.removal_requested() {
self.store_pool(Arc::clone(request));
}
}
pub fn remove_request(&mut self, request: &Arc<Request>) -> bool {
if let Some(pos) = self
.in_flight_requests
.iter()
.position(|r| Arc::ptr_eq(r, request))
{
self.in_flight_requests.remove(pos);
true
} else {
false
}
}
pub fn count_in_flight_request(&self) -> usize {
self.in_flight_requests.len()
}
pub fn count_pooled_request(&self) -> usize {
self.request_pool.len()
}
pub fn in_flight_requests(&self) -> &[Arc<Request>] {
&self.in_flight_requests
}
}
impl FileEntry {
pub fn find_faster_request(&mut self, base: &Arc<Request>) -> Option<Arc<Request>> {
self.sort_pool_by_speed();
if self.request_pool.is_empty() {
return None;
}
let now = Instant::now();
if now.duration_since(self.last_faster_replace) < STARTUP_IDLE_TIME {
return None;
}
let fastest = self.request_pool.first()?;
let fastest_stat = fastest.peer_stat()?;
let fastest_avg_speed = fastest_stat.avg_download_speed;
let base_stat = base.peer_stat();
let should_replace = match base_stat {
None => true,
Some(bs) => {
let base_speed = bs.download_speed;
(fastest_avg_speed as f64 * 0.8) > base_speed as f64
}
};
if should_replace {
let fastest_req = self.request_pool.remove(0);
self.in_flight_requests.push(Arc::clone(&fastest_req));
self.last_faster_replace = now;
return Some(fastest_req);
}
None
}
pub fn find_faster_request_by_server_stat(
&mut self,
base: &Arc<Request>,
used_hosts: &[(usize, String)],
server_stat_man: &ServerStatMan,
method: &str,
) -> Option<Arc<Request>> {
let now = Instant::now();
if now.duration_since(self.last_faster_replace) < STARTUP_IDLE_TIME {
return None;
}
let in_flight_hosts = self.collect_in_flight_hosts();
let base_stat = base.peer_stat();
let mut fast_cands: Vec<(u64, String)> = Vec::new();
for uri in self.remaining_uris.iter().take(NUM_URI_SCAN) {
let (host, protocol) = match extract_host_and_protocol(uri) {
Some(pair) => pair,
None => continue,
};
let host_count = in_flight_hosts.iter().filter(|h| *h == &host).count();
if host_count >= self.max_connection_per_server {
debug!(
"{} has already used {} times, not considered",
uri, self.max_connection_per_server
);
continue;
}
if used_hosts.iter().any(|(_, h)| h == &host) {
debug!("{} is in usedHosts, not considered", uri);
continue;
}
let stat = server_stat_man.find_stat_by_protocol(&host, &protocol);
let stat = match stat {
Some(s) => s,
None => continue,
};
if !stat.is_ok() {
continue;
}
let server_speed = stat.get_download_speed();
let is_faster = match base_stat {
Some(bs) => server_speed as f64 > bs.download_speed as f64 * 1.5,
None => server_speed > SPEED_THRESHOLD,
};
if is_faster {
fast_cands.push((server_speed, uri.clone()));
}
}
if fast_cands.is_empty() {
debug!("No faster server found.");
return None;
}
fast_cands.sort_by_key(|b| std::cmp::Reverse(b.0));
let (_, uri) = fast_cands.into_iter().next()?;
debug!("Selected {} from fastCands", uri);
let mut req = Request::new(&uri)?;
req.set_referer(base.referer());
req.set_method(method);
if let Some(pos) = self.remaining_uris.iter().position(|u| *u == uri) {
self.remaining_uris.remove(pos);
}
if !self.spent_uris.iter().any(|spent| spent == &uri) {
self.spent_uris.push_back(uri);
}
let req = Arc::new(req);
self.in_flight_requests.push(Arc::clone(&req));
self.last_faster_replace = now;
Some(req)
}
}
impl FileEntry {
pub(super) fn store_pool(&mut self, request: Arc<Request>) {
self.request_pool.push(request);
self.sort_pool_by_speed();
}
pub(super) fn sort_pool_by_speed(&mut self) {
self.request_pool.sort_by(|a, b| {
let a_speed = a.peer_stat().map(|ps| ps.avg_download_speed).unwrap_or(0);
let b_speed = b.peer_stat().map(|ps| ps.avg_download_speed).unwrap_or(0);
match b_speed.cmp(&a_speed) {
std::cmp::Ordering::Equal => {
Arc::as_ptr(a).cmp(&Arc::as_ptr(b))
}
other => other,
}
});
}
pub(super) fn collect_in_flight_hosts(&self) -> Vec<String> {
self.in_flight_requests
.iter()
.filter_map(|req| extract_host(req.uri()))
.collect()
}
pub(super) fn find_request_by_uri_in_flight(&self, uri: &str) -> Option<Arc<Request>> {
self.in_flight_requests
.iter()
.find(|req| !req.removal_requested() && req.uri() == uri)
.cloned()
}
pub(super) fn find_request_by_uri_in_pool(&self, uri: &str) -> Option<Arc<Request>> {
self.request_pool
.iter()
.find(|req| !req.removal_requested() && req.uri() == uri)
.cloned()
}
pub(super) fn get_request_with_in_flight_hosts(
&mut self,
selector: &dyn UriSelector,
uri_reuse: bool,
used_hosts: &[(usize, String)],
referer: &str,
method: &str,
in_flight_hosts: &[String],
) -> Option<Arc<Request>> {
let mut req: Option<Arc<Request>> = None;
for pass in 0..2 {
let mut pending: Vec<String> = Vec::new();
let mut ignore_host: Vec<String> = Vec::new();
while !self.remaining_uris.is_empty() {
let idx = selector.select(
&self.remaining_uris.iter().cloned().collect::<Vec<_>>(),
used_hosts,
);
let uri = match idx {
Some(i) if i < self.remaining_uris.len() => {
self.remaining_uris.remove(i).unwrap()
}
_ => break,
};
let mut new_req = match Request::new(&uri) {
Some(r) => r,
None => continue,
};
let host = new_req.host().to_owned();
let host_count = in_flight_hosts.iter().filter(|h| *h == &host).count();
if host_count >= self.max_connection_per_server {
pending.push(uri);
ignore_host.push(host);
continue;
}
if referer == "*" {
new_req.set_referer(&uri);
} else {
new_req.set_referer(referer);
}
new_req.set_method(method);
self.spent_uris.push_back(uri);
let req_arc = Arc::new(new_req);
self.in_flight_requests.push(Arc::clone(&req_arc));
req = Some(req_arc);
break;
}
for uri in pending.into_iter().rev() {
self.remaining_uris.push_front(uri);
}
if pass == 0 && uri_reuse && req.is_none() {
if self.remaining_uris.is_empty() {
self.reuse_uri(&ignore_host);
continue;
}
}
break;
}
req
}
}