use std::{collections::HashMap, fmt, sync::Arc, time::Duration};
use parking_lot::{Condvar, Mutex, RwLock};
#[cfg(not(target_arch = "wasm32"))]
use super::thread_pool::ThreadPool;
use super::{Load, LoadError, LoadState, LoadWaker, Loader, Source, link_meta::LinkMeta};
pub const DEFAULT_LINK_HEAD_BYTES: u64 = 512 * 1024;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum LinkPreviewError {
NotFound,
Denied,
TimedOut,
Server(u16),
Network(Arc<str>),
Unavailable(Arc<str>),
Other(Arc<str>),
}
impl LinkPreviewError {
pub fn detail(&self) -> Option<&str> {
match self {
Self::Network(detail) | Self::Unavailable(detail) | Self::Other(detail) => Some(detail),
_ => None,
}
}
pub fn of_load(error: &LoadError) -> Self {
match error {
LoadError::NotFound => Self::NotFound,
LoadError::Denied => Self::Denied,
LoadError::TimedOut => Self::TimedOut,
LoadError::Status(status) => Self::Server(*status),
LoadError::Unsupported(why) => Self::Unavailable(Arc::from(*why)),
LoadError::Io(why) => Self::Network(Arc::from(why.as_str())),
LoadError::TooLarge { .. } | LoadError::Cancelled => Self::Other(Arc::from(error.to_string())),
}
}
}
impl fmt::Display for LinkPreviewError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotFound => f.write_str("the page was not found"),
Self::Denied => f.write_str("the app does not allow this address"),
Self::TimedOut => f.write_str("the page took too long"),
Self::Server(status) => write!(f, "the server answered {status}"),
Self::Network(detail) | Self::Unavailable(detail) | Self::Other(detail) => f.write_str(detail),
}
}
}
impl std::error::Error for LinkPreviewError {}
#[derive(Clone, Debug, PartialEq)]
pub enum LinkFetch {
Loading,
Ready(Arc<LinkMeta>),
Failed(LinkPreviewError),
}
impl LinkFetch {
pub fn ready(meta: LinkMeta) -> Self {
Self::Ready(Arc::new(meta))
}
}
pub struct LinkRequest<'a> {
url: &'a str,
loader: Option<&'a Loader>,
wake: &'a LoadWaker,
}
impl<'a> LinkRequest<'a> {
pub fn new(url: &'a str, loader: Option<&'a Loader>, wake: &'a LoadWaker) -> Self {
Self { url, loader, wake }
}
pub fn url(&self) -> &str {
self.url
}
pub fn loader(&self) -> Option<&Loader> {
self.loader
}
pub fn wake(&self) -> &LoadWaker {
self.wake
}
}
pub trait LinkFetcher: Send + Sync {
fn fetch(&self, request: &LinkRequest<'_>) -> LinkFetch;
fn held(&self, url: &str) -> Option<LinkFetch> {
let _ = url;
None
}
}
impl<F: Fn(&str) -> LinkFetch + Send + Sync> LinkFetcher for F {
fn fetch(&self, request: &LinkRequest<'_>) -> LinkFetch {
self(request.url())
}
}
enum Kept {
Loading(Load),
Ended(LinkFetch),
}
pub struct PageFetcher {
head_bytes: RwLock<u64>,
kept: Mutex<HashMap<Arc<str>, Kept>>,
}
impl Default for PageFetcher {
fn default() -> Self {
Self::new()
}
}
impl PageFetcher {
pub fn new() -> Self {
Self {
head_bytes: RwLock::new(DEFAULT_LINK_HEAD_BYTES),
kept: Mutex::new(HashMap::new()),
}
}
pub fn head_bytes(&self, bytes: u64) -> &Self {
*self.head_bytes.write() = bytes.max(1);
self
}
pub fn forget(&self, url: &str) -> bool {
self.kept.lock().remove(url).is_some()
}
pub fn clear(&self) {
self.kept.lock().clear();
}
fn source(url: &str) -> Source {
Source::resource(url)
.header("Accept", "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5")
.header("User-Agent", "Mozilla/5.0 (compatible; lurq-link-preview)")
}
}
impl LinkFetcher for PageFetcher {
fn held(&self, url: &str) -> Option<LinkFetch> {
match self.kept.lock().get(url) {
Some(Kept::Ended(fetch)) => Some(fetch.clone()),
_ => None,
}
}
fn fetch(&self, request: &LinkRequest<'_>) -> LinkFetch {
let url = request.url();
let mut kept = self.kept.lock();
let load = match kept.get(url) {
Some(Kept::Ended(fetch)) => return fetch.clone(),
Some(Kept::Loading(load)) => load.clone(),
None => {
let Some(loader) = request.loader() else {
return LinkFetch::Failed(LinkPreviewError::Unavailable(Arc::from(
"the preview is in no app: nothing loads its page",
)));
};
let load = loader.load_start(&Self::source(url), *self.head_bytes.read());
kept.insert(Arc::from(url), Kept::Loading(load.clone()));
load
}
};
load.wake_with(request.wake().clone());
let ended = match load.state() {
LoadState::Loading { .. } => return LinkFetch::Loading,
LoadState::Ready(bytes) => LinkFetch::ready(LinkMeta::from_html(url, &bytes)),
LoadState::Failed(error) => LinkFetch::Failed(LinkPreviewError::of_load(&error)),
};
kept.insert(Arc::from(url), Kept::Ended(ended.clone()));
ended
}
}
static FETCHER: RwLock<Option<Arc<dyn LinkFetcher>>> = RwLock::new(None);
pub fn set_link_fetcher(fetcher: impl LinkFetcher + 'static) {
*FETCHER.write() = Some(Arc::new(fetcher));
}
pub fn reset_link_fetcher() {
*FETCHER.write() = None;
}
pub fn link_fetcher() -> Arc<dyn LinkFetcher> {
if let Some(fetcher) = FETCHER.read().clone() {
return fetcher;
}
static OWN: std::sync::OnceLock<Arc<PageFetcher>> = std::sync::OnceLock::new();
OWN.get_or_init(|| Arc::new(PageFetcher::new())).clone()
}
#[cfg(not(target_arch = "wasm32"))]
const ASKING_THREADS: usize = 4;
#[derive(Default)]
struct Asking {
counts: Mutex<Counts>,
answered: Condvar,
}
#[derive(Default)]
struct Counts {
pending: usize,
answered: usize,
}
thread_local! {
static ASKED_HERE: Arc<Asking> = Arc::new(Asking::default());
}
struct Answered(Arc<Asking>);
impl Drop for Answered {
fn drop(&mut self) {
let mut counts = self.0.counts.lock();
counts.pending = counts.pending.saturating_sub(1);
counts.answered += 1;
if counts.pending == 0 {
self.0.answered.notify_all();
}
}
}
pub(crate) fn fetch_off_thread(
fetcher: Arc<dyn LinkFetcher>,
url: Arc<str>,
loader: Option<Loader>,
wake: LoadWaker,
answered: impl FnOnce(LinkFetch) + Send + 'static,
) {
let asking = ASKED_HERE.with(Arc::clone);
asking.counts.lock().pending += 1;
let counted = Answered(asking);
run_off_thread(move || {
let _counted = counted;
let asked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
fetcher.fetch(&LinkRequest::new(&url, loader.as_ref(), &wake))
}));
answered(asked.unwrap_or_else(|_| {
tracing::error!(target: "lurq::resources", "a link fetcher panicked on {url}");
LinkFetch::Failed(LinkPreviewError::Other(Arc::from("the link fetcher panicked")))
}));
});
}
#[cfg(not(target_arch = "wasm32"))]
fn run_off_thread(job: impl FnOnce() + Send + 'static) {
static POOL: std::sync::OnceLock<ThreadPool> = std::sync::OnceLock::new();
POOL.get_or_init(|| ThreadPool::new(ASKING_THREADS)).execute(job);
}
#[cfg(target_arch = "wasm32")]
fn run_off_thread(job: impl FnOnce() + Send + 'static) {
wasm_bindgen_futures::spawn_local(async move { job() });
}
pub fn wait_link_fetches(timeout: Duration) -> Option<usize> {
let asking = ASKED_HERE.with(Arc::clone);
let mut counts = asking.counts.lock();
if !answered_in(&asking, &mut counts, timeout) {
return None;
}
Some(std::mem::take(&mut counts.answered))
}
#[cfg(not(target_arch = "wasm32"))]
fn answered_in(asking: &Asking, counts: &mut parking_lot::MutexGuard<'_, Counts>, timeout: Duration) -> bool {
let deadline = std::time::Instant::now() + timeout;
while counts.pending > 0 {
if asking.answered.wait_until(counts, deadline).timed_out() {
return counts.pending == 0;
}
}
true
}
#[cfg(target_arch = "wasm32")]
fn answered_in(_: &Asking, counts: &mut parking_lot::MutexGuard<'_, Counts>, _: Duration) -> bool {
counts.pending == 0
}
#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
use super::*;
fn waker() -> LoadWaker {
Arc::new(|| {})
}
#[test]
fn a_page_in_a_file_is_read_once_and_kept_by_its_address() {
let folder = std::env::temp_dir().join(format!("lurq-link-fetcher-{}", std::process::id()));
std::fs::create_dir_all(&folder).unwrap();
std::fs::write(
folder.join("page.html"),
b"<head><title>Kept</title><meta name=description content='Read once.'></head>",
)
.unwrap();
let loader = Loader::new();
loader.root(&folder);
let fetcher = PageFetcher::new();
let wake = waker();
let request = LinkRequest::new("page.html", Some(&loader), &wake);
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
let meta = loop {
match fetcher.fetch(&request) {
LinkFetch::Ready(meta) => break meta,
LinkFetch::Loading => {
assert!(std::time::Instant::now() < deadline, "the page does not come");
std::thread::sleep(std::time::Duration::from_millis(2));
}
LinkFetch::Failed(error) => panic!("{error}"),
}
};
assert_eq!(meta.title.as_deref(), Some("Kept"));
assert_eq!(fetcher.held("page.html"), Some(LinkFetch::Ready(meta.clone())));
assert_eq!(meta.description.as_deref(), Some("Read once."));
std::fs::remove_file(folder.join("page.html")).unwrap();
assert_eq!(fetcher.fetch(&request), LinkFetch::Ready(meta));
assert!(fetcher.forget("page.html"));
assert_eq!(fetcher.held("page.html"), None);
assert!(!fetcher.forget("page.html"));
}
#[test]
fn a_request_from_no_app_fails_and_a_function_answers_at_once() {
let wake = waker();
let request = LinkRequest::new("https://lurq.dev", None, &wake);
assert!(matches!(
PageFetcher::new().fetch(&request),
LinkFetch::Failed(LinkPreviewError::Unavailable(_))
));
let known = |url: &str| LinkFetch::ready(LinkMeta::new(url).title("Known"));
assert!(matches!(known.fetch(&request), LinkFetch::Ready(meta) if meta.title.as_deref() == Some("Known")));
}
#[test]
fn a_failed_load_is_its_kind() {
assert_eq!(
LinkPreviewError::of_load(&LoadError::NotFound),
LinkPreviewError::NotFound
);
assert_eq!(
LinkPreviewError::of_load(&LoadError::Status(503)),
LinkPreviewError::Server(503)
);
let network = LinkPreviewError::of_load(&LoadError::Io("reset".to_owned()));
assert_eq!(network.detail(), Some("reset"));
}
}