mod execution;
#[cfg(test)]
mod tests;
mod types;
pub use types::{select_mirrors_by_priority, try_mirrors_with_failover};
use std::sync::Arc;
use std::time::Duration;
use aria2_protocol::metalink::parser::UrlEntry;
use tracing::info;
use crate::download::{DownloadContext, file_entry::FileEntry};
use crate::error::{Aria2Error, FatalError, Result};
use crate::rate_limiter::RateLimiter;
use crate::request::request_group::{DownloadOptions, GroupId, RequestGroup};
use crate::util::rwlock_ext::RwLockRecover;
use types::FileDownloadInfo;
pub struct MetalinkFileInfo {
pub command: MetalinkDownloadCommand,
pub file_index: usize,
}
pub struct MetalinkDownloadCommand {
pub(crate) group: Arc<std::sync::RwLock<RequestGroup>>,
pub(crate) client: reqwest::Client,
pub(crate) output_path: std::path::PathBuf,
pub(crate) started: bool,
pub(crate) completed: bool,
pub(crate) completed_bytes: u64,
pub(crate) metalink_data: Vec<u8>,
pub(crate) file_info: Option<FileDownloadInfo>,
pub(crate) grouped_file_infos: Vec<(std::path::PathBuf, FileDownloadInfo)>,
pub(crate) checkpoint: Option<crate::engine::progress_checkpoint::ProgressCheckpoint>,
pub(crate) global_limiter: Option<RateLimiter>,
#[cfg(feature = "bittorrent")]
pub(crate) public_tracker_catalog:
Option<Arc<aria2_protocol::bittorrent::tracker::public_list::PublicTrackerList>>,
#[cfg(feature = "bittorrent")]
pub(crate) bt_registry: Option<Arc<std::sync::RwLock<crate::engine::bt_registry::BtRegistry>>>,
#[cfg(feature = "bittorrent")]
pub(crate) bt_listener: Option<Arc<crate::engine::bt_peer_listener::BtPeerListenerManager>>,
}
impl MetalinkDownloadCommand {
pub fn new(
gid: GroupId,
metalink_bytes: &[u8],
options: &DownloadOptions,
output_dir: Option<&str>,
) -> Result<Self> {
let doc = aria2_protocol::metalink::parser::MetalinkDocument::parse(metalink_bytes, None)
.map_err(|e| {
Aria2Error::Fatal(FatalError::Config(format!("Metalink parse failed: {}", e)))
})?;
if doc.files.is_empty() {
return Err(Aria2Error::Fatal(FatalError::Config(
"Metalink contains no files".into(),
)));
}
let file = &doc.files[0];
if file.urls.is_empty() {
let has_torrent_metaurl = file
.meta_urls
.iter()
.any(|m| m.mediatype == aria2_protocol::metalink::parser::MediaType::Torrent);
if !has_torrent_metaurl {
return Err(Aria2Error::Fatal(FatalError::Config(
"Metalink file has no download URL".into(),
)));
}
}
let dir = output_dir
.map(|d| d.to_string())
.or_else(|| options.dir.clone())
.unwrap_or_else(|| ".".to_string());
let filename = file.name.clone();
let path = std::path::PathBuf::from(&dir).join(&filename);
let urls: Vec<String> = file
.get_sorted_urls()
.iter()
.map(|u| u.url.clone())
.collect();
let group = RequestGroup::new(gid, urls, options.clone());
group.set_output_name(file.name.clone());
let client = build_http_client(options)?;
if doc.files.len() > 1 {
info!(
"MetalinkDownloadCommand created for first file of {}: {} -> {} ({} mirrors) \
[use create_multi_file() for all files]",
doc.files.len(),
file.name,
path.display(),
file.urls.len()
);
} else {
info!(
"MetalinkDownloadCommand created: {} -> {} ({} mirrors)",
file.name,
path.display(),
file.urls.len()
);
}
Ok(Self {
group: Arc::new(std::sync::RwLock::new(group)),
client,
output_path: path,
started: false,
completed: false,
completed_bytes: 0,
metalink_data: metalink_bytes.to_vec(),
file_info: None,
grouped_file_infos: Vec::new(),
checkpoint: None,
global_limiter: None,
#[cfg(feature = "bittorrent")]
public_tracker_catalog: None,
#[cfg(feature = "bittorrent")]
bt_registry: None,
#[cfg(feature = "bittorrent")]
bt_listener: None,
})
}
pub fn create_multi_file(
metalink_bytes: &[u8],
options: &DownloadOptions,
output_dir: Option<&str>,
gid_start: u64,
) -> Result<Vec<MetalinkFileInfo>> {
Self::create_multi_file_with_base_uri(metalink_bytes, options, output_dir, None, gid_start)
}
pub fn create_multi_file_with_base_uri(
metalink_bytes: &[u8],
options: &DownloadOptions,
output_dir: Option<&str>,
base_uri: Option<&str>,
gid_start: u64,
) -> Result<Vec<MetalinkFileInfo>> {
let doc =
aria2_protocol::metalink::parser::MetalinkDocument::parse(metalink_bytes, base_uri)
.map_err(|e| {
Aria2Error::Fatal(FatalError::Config(format!("Metalink parse failed: {}", e)))
})?;
if doc.files.is_empty() {
return Err(Aria2Error::Fatal(FatalError::Config(
"Metalink contains no files".into(),
)));
}
let dir = output_dir
.map(|d| d.to_string())
.or_else(|| options.dir.clone())
.unwrap_or_else(|| ".".to_string());
let client = build_http_client(options)?;
let mut commands = Vec::with_capacity(doc.files.len());
for (i, file) in doc.files.iter().enumerate() {
let has_torrent_metaurl = file.meta_urls.iter().any(|metaurl| {
metaurl.mediatype == aria2_protocol::metalink::parser::MediaType::Torrent
});
if file.urls.is_empty() && !has_torrent_metaurl {
tracing::debug!(
index = i,
name = %file.name,
"Skipping Metalink file with no downloadable resources"
);
continue;
}
let gid = GroupId::new(gid_start + i as u64);
let path = std::path::PathBuf::from(&dir).join(&file.name);
let sorted_urls: Vec<UrlEntry> = file
.get_sorted_urls()
.iter()
.map(|u| (*u).clone())
.collect();
let urls: Vec<String> = sorted_urls
.iter()
.filter(|u| u.is_non_p2p())
.map(|u| u.url.clone())
.collect();
let group = RequestGroup::new(gid, urls, options.clone());
group.set_output_name(file.name.clone());
let file_info = FileDownloadInfo {
expected_size: file.size,
hash_entry: file.strongest_hash().cloned(),
sorted_urls,
pieces: file.pieces.clone(),
torrent_metaurls: file
.meta_urls
.iter()
.filter(|m| m.mediatype == aria2_protocol::metalink::parser::MediaType::Torrent)
.cloned()
.collect(),
};
info!(
gid = gid.value(),
index = i,
name = %file.name,
path = %path.display(),
mirrors = file.urls.len(),
"Created MetalinkDownloadCommand for file"
);
commands.push(MetalinkFileInfo {
command: Self {
group: Arc::new(std::sync::RwLock::new(group)),
client: client.clone(),
output_path: path,
started: false,
completed: false,
completed_bytes: 0,
metalink_data: Vec::new(),
file_info: Some(file_info),
grouped_file_infos: Vec::new(),
checkpoint: None,
global_limiter: None,
#[cfg(feature = "bittorrent")]
public_tracker_catalog: None,
#[cfg(feature = "bittorrent")]
bt_registry: None,
#[cfg(feature = "bittorrent")]
bt_listener: None,
},
file_index: i,
});
}
Ok(commands)
}
pub fn create_multi_file_for_single(
file: &aria2_protocol::metalink::parser::MetalinkFile,
options: &DownloadOptions,
output_dir: Option<&str>,
gid_start: u64,
) -> Result<Vec<MetalinkFileInfo>> {
let dir = output_dir
.map(|d| d.to_string())
.or_else(|| options.dir.clone())
.unwrap_or_else(|| ".".to_string());
let client = build_http_client(options)?;
let torrent_metaurls: Vec<_> = file
.meta_urls
.iter()
.filter(|metaurl| {
metaurl.mediatype == aria2_protocol::metalink::parser::MediaType::Torrent
})
.cloned()
.collect();
if file.urls.is_empty() && torrent_metaurls.is_empty() {
return Ok(Vec::new());
}
let gid = GroupId::new(gid_start);
let path = std::path::PathBuf::from(&dir).join(&file.name);
let sorted_urls: Vec<UrlEntry> = file
.get_sorted_urls()
.iter()
.map(|u| (*u).clone())
.collect();
let urls: Vec<String> = sorted_urls
.iter()
.filter(|u| u.is_non_p2p())
.map(|u| u.url.clone())
.collect();
if urls.is_empty() && torrent_metaurls.is_empty() {
tracing::debug!(
name = %file.name,
"Skipping Metalink file with no downloadable resources"
);
return Ok(Vec::new());
}
let group = RequestGroup::new(gid, urls, options.clone());
group.set_output_name(file.name.clone());
let file_info = FileDownloadInfo {
expected_size: file.size,
hash_entry: file.strongest_hash().cloned(),
sorted_urls,
pieces: file.pieces.clone(),
torrent_metaurls,
};
info!(
gid = gid.value(),
name = %file.name,
path = %path.display(),
mirrors = file.urls.len(),
"Created MetalinkDownloadCommand for single file"
);
Ok(vec![MetalinkFileInfo {
command: Self {
group: Arc::new(std::sync::RwLock::new(group)),
client,
output_path: path,
started: false,
completed: false,
completed_bytes: 0,
metalink_data: Vec::new(),
file_info: Some(file_info),
grouped_file_infos: Vec::new(),
checkpoint: None,
global_limiter: None,
#[cfg(feature = "bittorrent")]
public_tracker_catalog: None,
#[cfg(feature = "bittorrent")]
bt_registry: None,
#[cfg(feature = "bittorrent")]
bt_listener: None,
},
file_index: 0,
}])
}
pub(crate) fn create_multi_file_group(
files: &[aria2_protocol::metalink::parser::MetalinkFile],
options: &DownloadOptions,
output_dir: Option<&str>,
gid: u64,
) -> Result<Self> {
if files.is_empty() {
return Err(Aria2Error::Fatal(FatalError::Config(
"Metalink group contains no files".to_string(),
)));
}
let dir = output_dir
.map(str::to_owned)
.or_else(|| options.dir.clone())
.unwrap_or_else(|| ".".to_string());
let client = build_http_client(options)?;
let mut entries = Vec::with_capacity(files.len());
let mut grouped_file_infos = Vec::with_capacity(files.len());
let mut offset = 0u64;
for file in files {
let sorted_urls: Vec<UrlEntry> = file.get_sorted_urls().into_iter().cloned().collect();
let urls: Vec<String> = sorted_urls
.iter()
.filter(|url| url.is_non_p2p())
.map(|url| url.url.clone())
.collect();
let torrent_metaurls = file
.meta_urls
.iter()
.filter(|metaurl| {
metaurl.mediatype == aria2_protocol::metalink::parser::MediaType::Torrent
})
.cloned()
.collect::<Vec<_>>();
if urls.is_empty() && torrent_metaurls.is_empty() {
continue;
}
let path = std::path::PathBuf::from(&dir).join(&file.name);
let mut entry = FileEntry::new(
path.to_string_lossy().into_owned(),
file.size.unwrap_or(0),
offset,
urls.clone(),
);
if let Some(original_name) = torrent_metaurls
.first()
.and_then(|metaurl| metaurl.name.clone())
{
entry.set_original_name(original_name);
}
entry.set_suffix_path(file.name.clone());
entry.set_max_connection_per_server(
options
.max_connection_per_server
.unwrap_or(crate::constants::DEFAULT_MAX_CONNECTION_PER_SERVER as u16)
.clamp(1, 16) as usize,
);
entry.set_unique_protocol(options.metalink_enable_unique_protocol);
offset = offset.saturating_add(file.size.unwrap_or(0));
entries.push(entry);
grouped_file_infos.push((
path,
FileDownloadInfo {
expected_size: file.size,
hash_entry: file.strongest_hash().cloned(),
sorted_urls,
pieces: file.pieces.clone(),
torrent_metaurls,
},
));
}
if grouped_file_infos.is_empty() {
return Err(Aria2Error::Fatal(FatalError::Config(
"Metalink group contains no downloadable files".to_string(),
)));
}
let group = RequestGroup::new(GroupId::new(gid), Vec::new(), options.clone());
let mut context = DownloadContext::new_default();
context.set_file_entries(entries);
group.set_download_context(Arc::new(context));
group.set_output_name(files[0].name.clone());
let output_path = grouped_file_infos[0].0.clone();
Ok(Self {
group: Arc::new(std::sync::RwLock::new(group)),
client,
output_path,
started: false,
completed: false,
completed_bytes: 0,
metalink_data: Vec::new(),
file_info: None,
grouped_file_infos,
checkpoint: None,
global_limiter: None,
#[cfg(feature = "bittorrent")]
public_tracker_catalog: None,
#[cfg(feature = "bittorrent")]
bt_registry: None,
#[cfg(feature = "bittorrent")]
bt_listener: None,
})
}
#[cfg(feature = "metalink")]
pub(crate) fn new_with_group_source(
group: Arc<std::sync::RwLock<RequestGroup>>,
metalink_bytes: &[u8],
file_index: usize,
options: &DownloadOptions,
base_uri: Option<&str>,
) -> Result<Self> {
let doc =
aria2_protocol::metalink::parser::MetalinkDocument::parse(metalink_bytes, base_uri)
.map_err(|e| {
Aria2Error::Fatal(FatalError::Config(format!("Metalink parse failed: {e}")))
})?;
let mut file = doc.files.get(file_index).cloned().ok_or_else(|| {
Aria2Error::Fatal(FatalError::Config(
"Metalink file index out of range".to_string(),
))
})?;
crate::engine::metalink_to_request_group::MetalinkToRequestGroup::new()
.normalize_file_for_runtime(&mut file, options);
let dir = options.dir.as_deref().unwrap_or(".");
let output_name = group
.recover()
.output_name()
.unwrap_or_else(|| file.name.clone());
let path = std::path::PathBuf::from(dir).join(output_name);
let sorted_urls: Vec<UrlEntry> = file.get_sorted_urls().into_iter().cloned().collect();
let file_info = FileDownloadInfo {
expected_size: file.size,
hash_entry: file.strongest_hash().cloned(),
pieces: file.pieces.clone(),
torrent_metaurls: file
.meta_urls
.iter()
.filter(|m| m.mediatype == aria2_protocol::metalink::parser::MediaType::Torrent)
.cloned()
.collect(),
sorted_urls,
};
Ok(Self {
group,
client: build_http_client(options)?,
output_path: path,
started: false,
completed: false,
completed_bytes: 0,
metalink_data: Vec::new(),
file_info: Some(file_info),
grouped_file_infos: Vec::new(),
checkpoint: None,
global_limiter: None,
#[cfg(feature = "bittorrent")]
public_tracker_catalog: None,
#[cfg(feature = "bittorrent")]
bt_registry: None,
#[cfg(feature = "bittorrent")]
bt_listener: None,
})
}
pub fn output_path(&self) -> &std::path::Path {
&self.output_path
}
pub fn set_global_limiter(&mut self, limiter: RateLimiter) {
self.global_limiter = Some(limiter);
}
#[cfg(feature = "bittorrent")]
pub fn set_public_tracker_catalog(
&mut self,
catalog: Arc<aria2_protocol::bittorrent::tracker::public_list::PublicTrackerList>,
) {
self.public_tracker_catalog = Some(catalog);
}
#[cfg(feature = "bittorrent")]
pub fn set_bt_registry(
&mut self,
registry: Arc<std::sync::RwLock<crate::engine::bt_registry::BtRegistry>>,
) {
self.bt_registry = Some(registry);
}
#[cfg(feature = "bittorrent")]
pub fn set_bt_listener(
&mut self,
listener: Arc<crate::engine::bt_peer_listener::BtPeerListenerManager>,
) {
self.bt_listener = Some(listener);
}
pub fn group(&self) -> std::sync::RwLockReadGuard<'_, RequestGroup> {
self.group.recover()
}
pub fn into_group(self) -> Arc<std::sync::RwLock<RequestGroup>> {
self.group
}
}
pub(crate) fn build_http_client(options: &DownloadOptions) -> Result<reqwest::Client> {
crate::http::client_pool::ensure_rustls_provider();
let client_tls = crate::http::client_identity::ClientTlsConfig::from_download_options(options);
let builder = reqwest::Client::builder()
.connect_timeout(Duration::from_secs(30))
.timeout(Duration::from_secs(300))
.gzip(options.http_accept_gzip)
.user_agent(crate::constants::USER_AGENT)
.redirect(reqwest::redirect::Policy::limited(5));
crate::http::client_identity::apply(builder, &client_tls)?
.build()
.map_err(|e| {
Aria2Error::Fatal(FatalError::Config(format!(
"HTTP client build failed: {}",
e
)))
})
}