use crate::{
services::{
arr::ArrApp,
putio::{self, PutIOTransfer},
},
state::OrphanFile,
AppData,
};
use actix_web::web::Data;
use anyhow::{Context, Result};
use async_channel::Sender;
use async_recursion::async_recursion;
use colored::*;
use log::{debug, error, info, warn};
use serde::{Deserialize, Serialize};
use std::{
collections::HashSet,
fmt::Display,
path::Path,
time::{Duration, Instant},
};
use tokio::time::sleep;
#[derive(Clone)]
pub struct Transfer {
pub name: String,
pub file_id: Option<i64>,
pub hash: Option<String>,
pub transfer_id: u64,
pub targets: Option<Vec<DownloadTarget>>,
pub app_data: Data<AppData>,
pub is_orphan: bool,
}
impl Transfer {
pub async fn is_imported(&self) -> bool {
let targets = self.targets.as_ref().unwrap().clone();
let apps: Vec<ArrApp> = self
.app_data
.config
.all_arrs()
.into_iter()
.map(|(name, kind, c)| ArrApp::new(name, kind, c))
.collect();
let targets = targets
.into_iter()
.filter(|t| t.target_type == TargetType::File)
.collect::<Vec<DownloadTarget>>();
if targets.is_empty() {
return false;
}
let mut results = Vec::<bool>::new();
for target in targets {
let mut service_results = vec![];
for app in &apps {
if let Some(mt) = &target.media_type {
if *mt != app.kind.media_type() {
continue;
}
}
let service_result = match app.check_imported(&target.to).await {
Ok(r) => r,
Err(e) => {
if self.app_data.state.should_log_arr_error(&app.name).await {
error!(
"Error retrieving history from {} (suppressing repeats for {:?}): {}",
app,
crate::state::StateManager::ARR_ERROR_LOG_INTERVAL,
e
);
}
false
}
};
if service_result {
info!(
"{}: found imported by {}",
target,
app.to_string().bright_blue()
);
}
service_results.push(service_result)
}
let any_imported = service_results.is_empty() || service_results.into_iter().any(|x| x);
results.push(any_imported);
}
results.into_iter().all(|x| x)
}
pub async fn get_download_targets(&self) -> Result<Vec<DownloadTarget>> {
self.generate_targets(None).await
}
pub async fn get_download_targets_in(&self, base_path: &str) -> Result<Vec<DownloadTarget>> {
self.generate_targets(Some(base_path.to_string())).await
}
async fn generate_targets(&self, base_path: Option<String>) -> Result<Vec<DownloadTarget>> {
info!("{}: generating targets", self);
let file_id = self.file_id.context("transfer has no file_id")?;
let default = "0000".to_string();
let hash = self.hash.as_ref().unwrap_or(&default).as_str();
recurse_download_targets(&self.app_data, file_id, hash, base_path, true).await
}
pub fn get_top_level(&self) -> DownloadTarget {
self.targets
.clone()
.unwrap()
.into_iter()
.find(|t| t.top_level)
.unwrap()
}
pub fn from(app_data: Data<AppData>, transfer: &PutIOTransfer) -> Self {
let default = &"Unknown".to_string();
let name = transfer.name.as_ref().unwrap_or(default);
Self {
transfer_id: transfer.id,
name: name.clone(),
file_id: transfer.file_id,
targets: None,
hash: transfer.hash.clone(),
app_data,
is_orphan: false,
}
}
pub fn from_orphan(app_data: Data<AppData>, file_id: i64, name: String) -> Self {
debug_assert!(file_id >= 0, "orphan file_id must be non-negative");
Self {
transfer_id: file_id as u64,
name,
file_id: Some(file_id),
targets: None,
hash: Some(format!("{:040x}", file_id as u64)),
app_data,
is_orphan: true,
}
}
}
impl Display for Transfer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let default = "0000".to_string();
let hash = &self.hash.as_ref().unwrap_or(&default)[..4];
let s = format!("[{}: {}]", hash, self.name).cyan();
write!(f, "{s}")
}
}
#[allow(clippy::double_must_use)] #[async_recursion]
async fn recurse_download_targets(
app_data: &Data<AppData>,
file_id: i64,
hash: &str,
override_base_path: Option<String>,
top_level: bool,
) -> Result<Vec<DownloadTarget>> {
let base_path = if let Some(path) = override_base_path {
path
} else {
app_data.state.get_download_dir_for_transfer(hash, &app_data.config.download_directory).await
};
let mut targets = Vec::<DownloadTarget>::new();
let response = putio::list_files(&app_data.config.putio.api_key, file_id).await?;
let to = Path::new(&base_path)
.join(&response.parent.name)
.to_string_lossy()
.to_string();
match response.parent.file_type.as_str() {
"FOLDER" => {
if !app_data
.config
.skip_directories
.contains(&response.parent.name.to_lowercase())
{
let new_base_path = to.clone();
targets.push(DownloadTarget {
from: None,
target_type: TargetType::Directory,
to,
top_level,
transfer_hash: hash.to_string(),
media_type: None,
});
for file in response.files {
targets.append(
&mut recurse_download_targets(
app_data,
file.id,
hash,
Some(new_base_path.clone()),
false,
)
.await?,
);
}
}
}
"VIDEO" | "AUDIO" => {
let url = putio::url(&app_data.config.putio.api_key, response.parent.id).await?;
targets.push(DownloadTarget {
from: Some(url),
target_type: TargetType::File,
to,
top_level,
transfer_hash: hash.to_string(),
media_type: MediaType::from_putio(response.parent.file_type.as_str()),
});
}
other => {
debug!(
"{}: skipping file type {}",
response.parent.name, other
);
}
}
Ok(targets)
}
#[derive(Clone)]
pub enum TransferMessage {
QueuedForDownload(Transfer),
Downloaded(Transfer),
Imported(Transfer),
}
#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Copy)]
pub enum MediaType {
Audio,
Video,
}
impl MediaType {
pub fn from_putio(file_type: &str) -> Option<Self> {
match file_type {
"AUDIO" => Some(Self::Audio),
"VIDEO" => Some(Self::Video),
_ => None,
}
}
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DownloadTarget {
pub from: Option<String>,
pub to: String,
pub target_type: TargetType,
pub top_level: bool,
pub transfer_hash: String,
pub media_type: Option<MediaType>,
}
impl Display for DownloadTarget {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let hash = &self.transfer_hash.as_str()[..4];
let s = format!("[{}: {}]", hash, self.to).magenta();
write!(f, "{s}")
}
}
#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
pub enum TargetType {
Directory,
File,
}
async fn is_managed(app_data: &Data<AppData>, putio_transfer: &PutIOTransfer) -> bool {
if app_data.config.download_unmanaged {
return true;
}
match &putio_transfer.hash {
Some(hash) => app_data.state.get_transfer(hash).await.is_some(),
None => false,
}
}
pub async fn produce_transfers(app_data: Data<AppData>, tx: Sender<TransferMessage>) -> Result<()> {
let putio_check_interval = Duration::from_secs(app_data.config.polling_interval);
let mut seen = Vec::<u64>::new();
let orphan_scan_interval =
Duration::from_secs(app_data.config.watch_folder_interval_secs.max(1));
let mut last_orphan_scan: Option<Instant> = None;
info!("Checking unfinished transfers");
for putio_transfer in &putio::list_transfers(&app_data.config.putio.api_key)
.await?
.transfers
{
let name = putio_transfer.name.clone().unwrap_or("??".to_string());
let mut transfer = Transfer::from(app_data.clone(), putio_transfer);
if putio_transfer.is_downloadable() && is_managed(&app_data, putio_transfer).await {
info!("Getting download target for {name}");
let targets = transfer.get_download_targets().await;
if targets.is_err() {
warn!("Could not get target for {name}");
continue;
}
transfer.targets = Some(targets?);
if transfer.is_imported().await {
info!("{}: already imported", transfer);
seen.push(transfer.transfer_id);
tx.send(TransferMessage::Imported(transfer)).await?;
} else {
info!("{}: not imported yet", transfer);
}
}
}
info!("Done checking for unfinished transfers. Starting to monitor transfers.");
let mut start = std::time::Instant::now();
loop {
if let Ok(list_transfer_response) =
putio::list_transfers(&app_data.config.putio.api_key).await
{
for putio_transfer in &list_transfer_response.transfers {
if seen.contains(&putio_transfer.id) || !putio_transfer.is_downloadable() {
continue;
}
let transfer = Transfer::from(app_data.clone(), putio_transfer);
if !is_managed(&app_data, putio_transfer).await {
debug!(
"{}: not managed by putioarr (no stored category/download-dir), skipping",
transfer
);
seen.push(putio_transfer.id);
continue;
}
info!("{}: ready for download", transfer);
tx.send(TransferMessage::QueuedForDownload(transfer))
.await?;
seen.push(putio_transfer.id);
}
let active_ids: Vec<u64> = list_transfer_response
.transfers
.iter()
.map(|t| t.id)
.collect();
seen.retain(|t| active_ids.contains(t));
if last_orphan_scan.is_none_or(|t| t.elapsed() >= orphan_scan_interval) {
scan_watch_folders(&app_data, &tx, &list_transfer_response.transfers).await;
last_orphan_scan = Some(Instant::now());
}
if start.elapsed().as_secs() >= 60 {
info!(
"Active transfers: {}",
list_transfer_response.transfers.len()
);
for t in &list_transfer_response.transfers {
let transfer = Transfer::from(app_data.clone(), t);
let pct = combined_progress_percent(&app_data, t).await;
info!(" {} ({}%)", transfer, pct);
}
start = std::time::Instant::now();
}
sleep(putio_check_interval).await;
} else {
warn!("List put.io transfers failed. Retrying..");
continue;
};
}
}
async fn combined_progress_percent(app_data: &Data<AppData>, t: &PutIOTransfer) -> u64 {
if app_data.state.is_local_complete(t.id).await {
return 100;
}
let size = t.size.unwrap_or(0).max(0) as u64;
if size == 0 {
return 0;
}
let putio_done = t.finished_at.is_some()
|| matches!(
t.status.to_uppercase().as_str(),
"SEEDING" | "COMPLETED" | "STOPPED"
);
if putio_done {
let local = match &t.hash {
Some(hash) => app_data
.state
.local_bytes_downloaded(hash)
.await
.unwrap_or(0)
.min(size),
None => 0,
};
(50 + 50 * local / size).min(99)
} else {
let downloaded = (t.downloaded.unwrap_or(0).max(0) as u64).min(size);
50 * downloaded / size
}
}
fn looks_like_episode(name: &str) -> bool {
let b = name.as_bytes();
let n = b.len();
let mut i = 0;
while i + 3 < n {
if (b[i] == b'S' || b[i] == b's') && b[i + 1].is_ascii_digit() {
let mut j = i + 1;
while j < n && b[j].is_ascii_digit() {
j += 1;
}
if j + 1 < n && (b[j] == b'E' || b[j] == b'e') && b[j + 1].is_ascii_digit() {
return true;
}
}
i += 1;
}
name.as_bytes()
.windows(6)
.any(|w| w.eq_ignore_ascii_case(b"season"))
}
async fn scan_watch_folders(
app_data: &Data<AppData>,
tx: &Sender<TransferMessage>,
active_transfers: &[PutIOTransfer],
) {
if app_data.config.watch_folders.is_empty() {
return;
}
let api_key = &app_data.config.putio.api_key;
let mut managed_active_file_ids: HashSet<i64> = HashSet::new();
for t in active_transfers {
if let Some(file_id) = t.file_id {
if is_managed(app_data, t).await {
managed_active_file_ids.insert(file_id);
}
}
}
for folder_id in &app_data.config.watch_folders {
let resp = match putio::list_files(api_key, *folder_id).await {
Ok(r) => r,
Err(e) => {
warn!("watch folder {}: listing failed: {}", folder_id, e);
continue;
}
};
for file in &resp.files {
if file.id < 0 || !matches!(file.file_type.as_str(), "FOLDER" | "VIDEO" | "AUDIO") {
continue;
}
if managed_active_file_ids.contains(&file.id) || app_data.state.has_orphan(file.id).await {
continue;
}
let category = if looks_like_episode(&file.name) {
app_data
.config
.sonarr
.as_ref()
.and_then(|c| c.category.clone())
} else {
app_data
.config
.radarr
.as_ref()
.and_then(|c| c.category.clone())
};
let download_dir = match &category {
Some(c) => format!("{}/{}", app_data.config.download_directory, c),
None => app_data.config.download_directory.clone(),
};
let mut transfer = Transfer::from_orphan(app_data.clone(), file.id, file.name.clone());
let targets = match transfer.get_download_targets_in(&download_dir).await {
Ok(t) if t.iter().any(|t| t.target_type == TargetType::File) => t,
Ok(_) => continue, Err(e) => {
warn!("{}: orphan target generation failed: {}", transfer, e);
continue;
}
};
transfer.targets = Some(targets);
if transfer.is_imported().await {
info!("{}: orphan already imported, deleting from put.io", transfer);
if let Err(e) = putio::delete_file(api_key, file.id).await {
warn!(
"{}: failed to delete imported orphan from put.io: {}",
transfer, e
);
}
continue;
}
info!("{}: orphan ready for download", transfer);
let hash = transfer.hash.clone().unwrap_or_default();
if tx
.send(TransferMessage::QueuedForDownload(transfer))
.await
.is_ok()
{
app_data
.state
.add_orphan(OrphanFile {
file_id: file.id,
name: file.name.clone(),
hash,
size: file.size,
download_dir,
})
.await;
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use figment::{
providers::{Format, Toml},
Figment,
};
fn app_data() -> Data<AppData> {
let config: crate::Config = Figment::new()
.merge(Toml::string(
r#"
bind_address = "0.0.0.0"
download_directory = "/downloads"
download_workers = 1
loglevel = "info"
orchestration_workers = 1
password = "p"
polling_interval = 10
port = 9091
skip_directories = ["sample"]
uid = 1000
username = "u"
[putio]
api_key = "k"
"#,
))
.extract()
.unwrap();
Data::new(AppData {
config,
state: crate::state::StateManager::new("k".to_string()),
http: reqwest::Client::new(),
})
}
fn target(target_type: TargetType, to: &str) -> DownloadTarget {
DownloadTarget {
from: None,
to: to.to_string(),
target_type,
top_level: false,
transfer_hash: "0".repeat(40),
media_type: Some(MediaType::Video),
}
}
#[actix_rt::test]
async fn transfer_with_only_folders_is_not_imported() {
let mut t = Transfer::from_orphan(app_data(), 1, "Some.Release".to_string());
t.targets = Some(vec![target(TargetType::Directory, "/downloads/Some.Release")]);
assert!(!t.is_imported().await);
}
#[actix_rt::test]
async fn file_with_no_arr_to_ask_still_counts_as_imported() {
let mut t = Transfer::from_orphan(app_data(), 1, "Some.Release".to_string());
t.targets = Some(vec![
target(TargetType::Directory, "/downloads/Some.Release"),
target(TargetType::File, "/downloads/Some.Release/movie.mkv"),
]);
assert!(t.is_imported().await);
}
}