use std::collections::VecDeque;
use std::sync::Arc;
use tracing::debug;
use super::entry::FileEntry;
use super::helpers::{extract_host, is_valid_uri};
use super::types::UriResult;
impl FileEntry {
pub fn remaining_uris(&self) -> &VecDeque<String> {
&self.remaining_uris
}
pub fn remaining_uris_mut(&mut self) -> &mut VecDeque<String> {
&mut self.remaining_uris
}
pub fn spent_uris(&self) -> &VecDeque<String> {
&self.spent_uris
}
pub fn uris(&self) -> Vec<String> {
self.spent_uris
.iter()
.chain(self.remaining_uris.iter())
.cloned()
.collect()
}
pub fn set_uris(&mut self, uris: &[String]) -> usize {
self.remaining_uris.clear();
self.add_uris(uris)
}
pub fn add_uris(&mut self, uris: &[String]) -> usize {
uris.iter().filter(|uri| self.add_uri(uri)).count()
}
pub fn add_uri(&mut self, uri: &str) -> bool {
if is_valid_uri(uri) {
self.remaining_uris.push_back(uri.to_owned());
true
} else {
false
}
}
pub fn insert_uri(&mut self, uri: &str, pos: usize) -> bool {
if !is_valid_uri(uri) {
return false;
}
let insert_pos = pos.min(self.remaining_uris.len());
if insert_pos == self.remaining_uris.len() {
self.remaining_uris.push_back(uri.to_owned());
} else if insert_pos == 0 {
self.remaining_uris.push_front(uri.to_owned());
} else {
let mut right = self.remaining_uris.split_off(insert_pos);
self.remaining_uris.push_back(uri.to_owned());
self.remaining_uris.append(&mut right);
}
true
}
pub fn remove_uri(&mut self, uri: &str) -> bool {
if let Some(pos) = self.remaining_uris.iter().position(|u| u == uri) {
self.remaining_uris.remove(pos);
return true;
}
if let Some(pos) = self.spent_uris.iter().position(|u| u == uri) {
self.spent_uris.remove(pos);
let req = self
.find_request_by_uri_in_flight(uri)
.or_else(|| self.find_request_by_uri_in_pool(uri));
if let Some(req) = req {
if let Some(pos) = self.request_pool.iter().position(|r| Arc::ptr_eq(r, &req)) {
self.request_pool.remove(pos);
}
}
return true;
}
false
}
pub fn remove_uri_and_mark(&mut self, uri: &str) -> bool {
if let Some(pos) = self.remaining_uris.iter().position(|u| u == uri) {
self.remaining_uris.remove(pos);
return true;
}
if let Some(pos) = self.spent_uris.iter().position(|u| u == uri) {
self.spent_uris.remove(pos);
if let Some(req) = self.find_request_by_uri_in_flight(uri) {
if let Some(pool_pos) = self.request_pool.iter().position(|r| Arc::ptr_eq(r, &req))
{
self.request_pool.remove(pool_pos);
}
} else if let Some(req) = self.find_request_by_uri_in_pool(uri) {
if let Some(pool_pos) = self.request_pool.iter().position(|r| Arc::ptr_eq(r, &req))
{
self.request_pool.remove(pool_pos);
}
}
return true;
}
false
}
pub fn remove_uri_whose_hostname_is(&mut self, hostname: &str) {
let before = self.remaining_uris.len();
self.remaining_uris
.retain(|uri| extract_host(uri).as_deref() != Some(hostname));
let removed = before - self.remaining_uris.len();
if removed > 0 {
debug!(
"Removed {} URIs with hostname '{}' for path={}",
removed, hostname, self.path
);
}
}
pub fn remove_identical_uri(&mut self, uri: &str) {
self.remaining_uris.retain(|u| u != uri);
}
pub fn empty_request_uri(&self) -> bool {
self.remaining_uris.is_empty()
&& self.in_flight_requests.is_empty()
&& self.request_pool.is_empty()
}
}
impl FileEntry {
pub fn add_uri_result(&mut self, uri: String, result_code: u16) {
self.uri_results.push_back(UriResult::new(uri, result_code));
}
pub fn uri_results(&self) -> &VecDeque<UriResult> {
&self.uri_results
}
pub fn extract_uri_result(&mut self, res: &mut VecDeque<UriResult>, result_code: u16) {
let mut matching = VecDeque::new();
let mut non_matching = VecDeque::new();
for ur in self.uri_results.drain(..) {
if ur.result_code == result_code {
matching.push_back(ur);
} else {
non_matching.push_back(ur);
}
}
res.extend(matching);
self.uri_results = non_matching;
}
}
impl FileEntry {
pub fn reuse_uri(&mut self, ignore: &[String]) {
for host in ignore {
debug!("ignore host={}", host);
}
let mut spent_sorted: Vec<String> = self.spent_uris.iter().cloned().collect();
spent_sorted.sort();
spent_sorted.dedup();
let mut error_uris: Vec<String> = self.uri_results.iter().map(|r| r.uri.clone()).collect();
error_uris.sort();
error_uris.dedup();
for uri in &error_uris {
debug!("error URI={}", uri);
}
let mut reusable_uris = Vec::new();
let mut error_iter = error_uris.iter().peekable();
for spent_uri in &spent_sorted {
while error_iter.peek().is_some_and(|e| *e < spent_uri) {
error_iter.next();
}
if error_iter.peek() == Some(&spent_uri) {
error_iter.next();
continue;
}
reusable_uris.push(spent_uri.clone());
}
reusable_uris.retain(|uri| {
extract_host(uri)
.as_ref()
.is_none_or(|host| !ignore.iter().any(|ig| ig == host.as_str()))
});
debug!("Found {} reusable URIs", reusable_uris.len());
for uri in &reusable_uris {
debug!("URI={}", uri);
}
self.remaining_uris.extend(reusable_uris);
}
pub fn put_back_request(&mut self) {
for req in self.in_flight_requests.iter().rev() {
self.remaining_uris.push_front(req.uri().to_owned());
}
for req in self.request_pool.iter().rev() {
self.remaining_uris.push_front(req.uri().to_owned());
}
}
}