use std::collections::HashMap;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use bytes::Bytes;
use futures::Stream;
use futures::stream::{self, StreamExt};
use tokio::sync::{Mutex, Notify, watch};
use tokio::time::sleep;
use tracing::{info, warn};
mod parse;
mod source_fetcher;
#[cfg(test)]
mod tests;
use parse::EMBEDDED_TRACKER_LIST;
use source_fetcher::{ReqwestTrackerSourceFetcher, fetch_tracker_source, tracker_source_origin};
pub const DEFAULT_TRACKER_SOURCE: &str = "https://cf.trackerslist.com/best.txt";
pub const DEFAULT_TRACKER_UPDATE_INTERVAL: Duration = Duration::from_secs(604_800);
pub const DEFAULT_TRACKER_SOURCE_TIMEOUT: Duration = Duration::from_secs(15);
pub const MAX_TRACKER_SOURCE_BYTES: usize = 8 * 1024 * 1024;
pub const MAX_TRACKERS_PER_SOURCE: usize = 4096;
pub const MAX_TRACKER_SOURCE_CONCURRENCY: usize = 4;
const MAX_BACKOFF: Duration = Duration::from_secs(3_600);
const REMOTE_TEMPORARY_BACKOFF: Duration = Duration::from_secs(30);
pub async fn read_tracker_source_body<S, E>(mut chunks: S) -> Result<String, String>
where
S: Stream<Item = Result<Bytes, E>> + Unpin,
E: std::fmt::Display,
{
let mut body = Vec::new();
while let Some(chunk) = chunks.next().await {
let chunk = chunk.map_err(|error| format!("Read body failed: {error}"))?;
if chunk.len() > MAX_TRACKER_SOURCE_BYTES.saturating_sub(body.len()) {
return Err(format!(
"Tracker source response exceeds the {MAX_TRACKER_SOURCE_BYTES}-byte limit"
));
}
body.extend_from_slice(&chunk);
}
Ok(String::from_utf8_lossy(&body).into_owned())
}
pub trait TrackerSourceFetcher: Send + Sync {
fn fetch<'a>(
&'a self,
source: &'a str,
timeout: Duration,
) -> Pin<Box<dyn Future<Output = Result<String, String>> + Send + 'a>>;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TrackerProtocol {
Http,
Https,
Udp,
Wss,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TrackerFailureKind {
Network,
Timeout,
RemoteTemporary,
TrackerRejected,
MalformedResponse,
}
impl TrackerFailureKind {
pub fn as_str(self) -> &'static str {
match self {
Self::Network => "network",
Self::Timeout => "timeout",
Self::RemoteTemporary => "remoteTemporary",
Self::TrackerRejected => "trackerRejected",
Self::MalformedResponse => "malformedResponse",
}
}
}
impl TrackerProtocol {
pub fn as_str(&self) -> &'static str {
match self {
Self::Http => "http",
Self::Https => "https",
Self::Udp => "udp",
Self::Wss => "wss",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TrackerEntry {
pub url: String,
pub protocol: TrackerProtocol,
pub host: String,
pub port: u16,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TrackerCatalogConfig {
pub enabled: bool,
pub sources: Vec<String>,
pub update_interval: Duration,
}
impl Default for TrackerCatalogConfig {
fn default() -> Self {
Self {
enabled: true,
sources: vec![DEFAULT_TRACKER_SOURCE.to_string()],
update_interval: DEFAULT_TRACKER_UPDATE_INTERVAL,
}
}
}
pub struct PublicTrackerListStats {
pub total_entries: usize,
pub http_count: usize,
pub udp_count: usize,
pub wss_count: usize,
pub is_embedded_fallback: bool,
pub last_updated: Option<Duration>,
pub sources: Vec<String>,
pub disabled: bool,
}
#[derive(Debug, Clone, Default)]
struct TrackerHealth {
consecutive_failures: u32,
next_retry: Option<Instant>,
successes: u64,
failures: u64,
}
pub struct PublicTrackerList {
entries: tokio::sync::RwLock<Arc<Vec<TrackerEntry>>>,
source_entries: tokio::sync::RwLock<HashMap<String, Vec<TrackerEntry>>>,
source_fetcher: tokio::sync::RwLock<Arc<dyn TrackerSourceFetcher>>,
config: tokio::sync::RwLock<TrackerCatalogConfig>,
source_request_timeout: tokio::sync::RwLock<Duration>,
source_fetch_concurrency: AtomicUsize,
health: Mutex<HashMap<String, TrackerHealth>>,
last_updated: tokio::sync::RwLock<Option<Instant>>,
last_refresh_error: tokio::sync::RwLock<Option<String>>,
running: AtomicBool,
update_started: AtomicBool,
config_changed: Notify,
updates: watch::Sender<u64>,
}
impl Default for PublicTrackerList {
fn default() -> Self {
Self::new()
}
}
impl PublicTrackerList {
pub fn new() -> Self {
Self::new_with_config(TrackerCatalogConfig::default())
}
pub fn new_with_config(config: TrackerCatalogConfig) -> Self {
Self::new_with_config_and_fetcher(config, Arc::new(ReqwestTrackerSourceFetcher::new()))
}
pub fn new_with_config_and_fetcher(
config: TrackerCatalogConfig,
source_fetcher: Arc<dyn TrackerSourceFetcher>,
) -> Self {
let entries = Self::parse(EMBEDDED_TRACKER_LIST);
let (updates, _) = watch::channel(0);
Self {
entries: tokio::sync::RwLock::new(Arc::new(entries)),
source_entries: tokio::sync::RwLock::new(HashMap::new()),
source_fetcher: tokio::sync::RwLock::new(source_fetcher),
config: tokio::sync::RwLock::new(config),
source_request_timeout: tokio::sync::RwLock::new(DEFAULT_TRACKER_SOURCE_TIMEOUT),
source_fetch_concurrency: AtomicUsize::new(MAX_TRACKER_SOURCE_CONCURRENCY),
health: Mutex::new(HashMap::new()),
last_updated: tokio::sync::RwLock::new(Some(Instant::now())),
last_refresh_error: tokio::sync::RwLock::new(None),
running: AtomicBool::new(true),
update_started: AtomicBool::new(false),
config_changed: Notify::new(),
updates,
}
}
pub fn subscribe_updates(&self) -> watch::Receiver<u64> {
self.updates.subscribe()
}
fn notify_updated(&self) {
self.updates
.send_modify(|revision| *revision = revision.wrapping_add(1));
}
pub fn parse(text: &str) -> Vec<TrackerEntry> {
parse::parse_tracker_list(text)
}
pub async fn config(&self) -> TrackerCatalogConfig {
self.config.read().await.clone()
}
pub async fn source_request_timeout(&self) -> Duration {
*self.source_request_timeout.read().await
}
pub fn set_source_fetch_concurrency_limit(&self, limit: usize) {
self.source_fetch_concurrency.store(
limit.clamp(1, MAX_TRACKER_SOURCE_CONCURRENCY),
Ordering::Release,
);
}
pub fn source_fetch_concurrency_limit(&self) -> usize {
self.source_fetch_concurrency.load(Ordering::Acquire)
}
pub async fn set_source_request_timeout(&self, timeout: Duration) {
*self.source_request_timeout.write().await = if timeout.is_zero() {
DEFAULT_TRACKER_SOURCE_TIMEOUT
} else {
timeout
};
self.notify_updated();
self.config_changed.notify_one();
}
pub fn set_source_request_timeout_now(&self, timeout: Duration) {
*self
.source_request_timeout
.try_write()
.expect("tracker source timeout must not be held during engine setup") =
if timeout.is_zero() {
DEFAULT_TRACKER_SOURCE_TIMEOUT
} else {
timeout
};
self.notify_updated();
self.config_changed.notify_one();
}
pub async fn set_config(&self, mut config: TrackerCatalogConfig) {
config.sources = normalize_sources(config.sources);
if config.update_interval.is_zero() {
config.update_interval = DEFAULT_TRACKER_UPDATE_INTERVAL;
}
*self.config.write().await = config;
self.notify_updated();
self.config_changed.notify_one();
}
pub fn set_config_now(&self, mut config: TrackerCatalogConfig) {
config.sources = normalize_sources(config.sources);
if config.update_interval.is_zero() {
config.update_interval = DEFAULT_TRACKER_UPDATE_INTERVAL;
}
*self
.config
.try_write()
.expect("public tracker config must not be held during engine setup") = config;
self.notify_updated();
self.config_changed.notify_one();
}
pub async fn sources(&self) -> Vec<String> {
self.config.read().await.sources.clone()
}
pub async fn contains(&self, url: &str) -> bool {
self.snapshot().await.iter().any(|entry| entry.url == url)
}
pub async fn snapshot(&self) -> Arc<Vec<TrackerEntry>> {
self.entries.read().await.clone()
}
pub async fn available_snapshot(&self) -> Arc<Vec<TrackerEntry>> {
if !self.config.read().await.enabled {
return Arc::new(Vec::new());
}
let entries = self.snapshot().await;
let now = Instant::now();
let health = self.health.lock().await;
Arc::new(
entries
.iter()
.filter(|entry| {
health
.get(&entry.url)
.and_then(|state| state.next_retry)
.is_none_or(|retry_at| retry_at <= now)
})
.cloned()
.collect(),
)
}
pub async fn get_http_trackers(&self) -> Vec<String> {
let entries = self.available_snapshot().await;
entries
.iter()
.filter(|e| e.protocol == TrackerProtocol::Http || e.protocol == TrackerProtocol::Https)
.map(|e| e.url.clone())
.collect()
}
pub async fn get_udp_trackers(&self) -> Vec<String> {
let entries = self.available_snapshot().await;
entries
.iter()
.filter(|e| e.protocol == TrackerProtocol::Udp)
.map(|e| e.url.clone())
.collect()
}
pub async fn get_all(&self) -> Vec<TrackerEntry> {
self.snapshot().await.as_ref().clone()
}
pub async fn fetch_and_update(&self, url: &str) -> Result<usize, String> {
let timeout = self.source_request_timeout().await;
let fetcher = self.source_fetcher.read().await.clone();
let entries = fetch_tracker_source(fetcher.as_ref(), url, timeout).await?;
self.source_entries
.write()
.await
.insert(url.to_string(), entries);
self.merge_source_snapshots().await
}
pub fn start_auto_update(self: &Arc<Self>, url: String, interval: Duration) {
self.set_config_now(TrackerCatalogConfig {
enabled: true,
sources: vec![url],
update_interval: interval,
});
self.start_catalog_update();
}
pub fn start_catalog_update(self: &Arc<Self>) {
if self.update_started.swap(true, Ordering::AcqRel) {
return;
}
let catalog = Arc::clone(self);
tokio::spawn(async move {
while catalog.running.load(Ordering::Relaxed) {
let config = catalog.config().await;
if config.enabled
&& let Err(error) = catalog.refresh().await
{
warn!("Public tracker catalog refresh failed: {}", error);
}
let sleep = sleep(config.update_interval);
tokio::pin!(sleep);
tokio::select! {
_ = &mut sleep => {},
_ = catalog.config_changed.notified() => {},
}
}
info!("Public tracker catalog update loop exited");
});
}
pub async fn refresh(&self) -> Result<usize, String> {
let config = self.config().await;
if !config.enabled {
return Ok(0);
}
let sources = normalize_sources(config.sources);
if sources.is_empty() {
*self.last_refresh_error.write().await = None;
return Ok(0);
}
let fetcher = self.source_fetcher.read().await.clone();
let request_timeout = self.source_request_timeout().await;
let source_concurrency = self.source_fetch_concurrency.load(Ordering::Acquire);
let results = stream::iter(sources.iter().cloned().map(|source| {
let fetcher = Arc::clone(&fetcher);
async move {
let result = fetch_tracker_source(fetcher.as_ref(), &source, request_timeout).await;
(source, result)
}
}))
.buffer_unordered(source_concurrency)
.collect::<Vec<_>>()
.await;
let mut errors = Vec::new();
let mut source_entries = self.source_entries.write().await;
source_entries.retain(|source, _| sources.contains(source));
for (source, result) in results {
match result {
Ok(entries) => {
source_entries.insert(source, entries);
}
Err(error) => errors.push(format!("{}: {}", tracker_source_origin(&source), error)),
}
}
let has_source_snapshot = !source_entries.is_empty();
drop(source_entries);
if !has_source_snapshot {
let error = errors.join("; ");
*self.last_refresh_error.write().await = Some(error.clone());
return Err(if error.is_empty() {
"All public tracker sources returned no valid trackers".to_string()
} else {
error
});
}
let count = self.merge_source_snapshots().await?;
*self.last_refresh_error.write().await = (!errors.is_empty()).then_some(errors.join("; "));
Ok(count)
}
async fn merge_source_snapshots(&self) -> Result<usize, String> {
let source_entries = self.source_entries.read().await;
let mut merged = Vec::new();
let mut seen = std::collections::HashSet::new();
for entries in source_entries.values() {
for entry in entries {
if seen.insert(entry.url.clone()) {
merged.push(entry.clone());
}
}
}
drop(source_entries);
self.replace_snapshot(merged).await
}
async fn replace_snapshot(&self, entries: Vec<TrackerEntry>) -> Result<usize, String> {
if entries.is_empty() {
return Err("Parsed 0 entries from response".to_string());
}
let count = entries.len();
*self.entries.write().await = Arc::new(entries);
let current_urls: std::collections::HashSet<_> = self
.entries
.read()
.await
.iter()
.map(|entry| entry.url.clone())
.collect();
self.health
.lock()
.await
.retain(|url, _| current_urls.contains(url));
*self.last_updated.write().await = Some(Instant::now());
self.notify_updated();
info!("Public tracker catalog updated: {} trackers", count);
Ok(count)
}
pub async fn record_success(&self, url: &str) {
let mut health = self.health.lock().await;
let state = health.entry(url.to_string()).or_default();
state.consecutive_failures = 0;
state.next_retry = None;
state.successes = state.successes.saturating_add(1);
}
pub async fn record_failure(&self, url: &str) {
self.record_failure_kind(url, TrackerFailureKind::RemoteTemporary)
.await;
}
pub async fn record_failure_kind(&self, url: &str, kind: TrackerFailureKind) {
let mut health = self.health.lock().await;
let state = health.entry(url.to_string()).or_default();
state.failures = state.failures.saturating_add(1);
let (consecutive_failures, base_backoff) = match kind {
TrackerFailureKind::Network | TrackerFailureKind::Timeout => return,
TrackerFailureKind::RemoteTemporary => (
state.consecutive_failures.saturating_add(1),
REMOTE_TEMPORARY_BACKOFF,
),
TrackerFailureKind::MalformedResponse => return,
TrackerFailureKind::TrackerRejected => return,
};
state.consecutive_failures = consecutive_failures;
let exponent = consecutive_failures.saturating_sub(1).min(10);
let backoff = base_backoff
.checked_mul(1u32 << exponent)
.unwrap_or(MAX_BACKOFF)
.min(MAX_BACKOFF);
state.next_retry = Some(Instant::now() + backoff);
}
pub async fn last_refresh_error(&self) -> Option<String> {
self.last_refresh_error.read().await.clone()
}
pub fn shutdown(&self) {
self.running.store(false, Ordering::Relaxed);
self.config_changed.notify_one();
}
pub async fn stats(&self) -> PublicTrackerListStats {
let entries = self.snapshot().await;
let http_count = entries
.iter()
.filter(|e| matches!(e.protocol, TrackerProtocol::Http | TrackerProtocol::Https))
.count();
let udp_count = entries
.iter()
.filter(|e| e.protocol == TrackerProtocol::Udp)
.count();
let wss_count = entries
.iter()
.filter(|e| e.protocol == TrackerProtocol::Wss)
.count();
let embedded = EMBEDDED_TRACKER_LIST
.lines()
.filter(|l| l.trim().contains("://"))
.count();
PublicTrackerListStats {
total_entries: entries.len(),
http_count,
udp_count,
wss_count,
is_embedded_fallback: entries.len() == embedded,
last_updated: self.last_updated.read().await.map(|t| t.elapsed()),
sources: self.sources().await,
disabled: !self.config.read().await.enabled,
}
}
}
fn normalize_sources(sources: Vec<String>) -> Vec<String> {
let mut seen = std::collections::HashSet::new();
sources
.into_iter()
.flat_map(|source| {
source
.split([',', '\n'])
.map(str::trim)
.map(str::to_string)
.collect::<Vec<_>>()
})
.filter(|source| !source.is_empty() && seen.insert(source.clone()))
.collect()
}