use chrono::{DateTime, SubsecRound, Utc};
use indicatif::ProgressBar;
use lol_html::element;
use lol_html::{RewriteStrSettings, rewrite_str};
use rapidhash::RapidHashMap as HashMap;
use rapidhash::RapidHashSet as HashSet;
use std::error::Error;
use std::fs;
use std::path::Path;
use const_format::formatcp;
use anyhow::anyhow;
use bytes::Buf;
use epub_builder::{EpubBuilder, EpubContent, ReferenceType, ZipLibrary};
use rand::{Rng, SeedableRng};
use reqwest::Client;
use slug::slugify;
use uuid::Uuid;
use crate::html::sequence_title_page;
use crate::{
cache::get_crosspost,
cli::CacheOptions,
html::{cover_page, title_page, wrap_body},
image::{cover_image, intern_image},
};
use lesspub_schema::{Collection, Post, Sequence};
const LW_IMG_PREFIX: &str = "https://res.cloudinary.com/lesswrong-2-0/image/upload/c_fill,ar_0.625,g_auto:subject:thirds_0,q_auto:best/";
const SEQUENCE_IMG_PREFIX: &str = formatcp!("{LW_IMG_PREFIX}v1/");
const DEFAULT_SEQUENCE_IMG: &str = "sequences/vnyzzznenju0hzdv6pqb.jpg";
const DEFAULT_POST_IMG_URL: &str = formatcp!(
"{LW_IMG_PREFIX}ohabryka_Topographic_aquarelle_book_cover_by_Thomas_W._Schaller_f9c9dbbe-4880-4f12-8ebb-b8f0b900abc1_m4k6dy_734413"
);
const COVER_PATH: &str = "cover.png";
const SEQUENCE_URL_PREFIX: &str = "https://www.lesswrong.com/sequences/";
const LW_POST_URL_PREFIX: &str = "https://www.lesswrong.com/posts/";
#[derive(Debug, Clone)]
pub struct EpubInfo {
pub created: Option<DateTime<Utc>>,
pub updated: Option<DateTime<Utc>>,
pub entries: usize,
}
pub fn get_saved_epub_data(
title: &str,
output_dir: &Path,
) -> Result<Option<EpubInfo>, Box<dyn Error>> {
let filename = format!("{}.epub", slugify(title));
let path = output_dir.join(filename);
if !path.exists() || !path.is_file() {
Ok(None)
} else if let Ok(epub) = epub::doc::EpubDoc::new(path) {
Ok(Some(EpubInfo {
created: epub
.mdata("date")
.and_then(|meta| Some(DateTime::parse_from_rfc3339(&meta).ok()?.to_utc())),
updated: epub
.mdata("dcterms:modified")
.and_then(|meta| Some(DateTime::parse_from_rfc3339(&meta).ok()?.to_utc())),
entries: epub
.toc
.len()
.max(epub.toc.into_iter().flat_map(|p| p.children).count()),
}))
} else {
Ok(None)
}
}
pub enum EpubStatus {
UpToDate,
OutOfDate,
Nonexistent,
}
pub fn parse_saved_post_epub_data(
data: Result<Option<EpubInfo>, Box<dyn Error>>,
post: &Post,
) -> EpubStatus {
match data {
Ok(Some(epub_info)) => {
if epub_info.updated.is_some_and(|updated| {
post.modifiedAt
.is_some_and(|modified_at| updated < modified_at.trunc_subsecs(0))
}) {
EpubStatus::OutOfDate
} else {
EpubStatus::UpToDate
}
}
_ => EpubStatus::Nonexistent,
}
}
pub fn parse_saved_sequence_epub_data(
data: Result<Option<EpubInfo>, Box<dyn Error>>,
sequence: &Sequence,
) -> EpubStatus {
match data {
Ok(Some(epub_info)) => {
if epub_info.entries < sequence.postsCount.unwrap_or_default() as usize
|| epub_info.updated.is_some_and(|updated| {
updated < sequence.lastUpdated.unwrap_or_default().trunc_subsecs(0)
})
{
EpubStatus::OutOfDate
} else {
EpubStatus::UpToDate
}
}
_ => EpubStatus::Nonexistent,
}
}
pub async fn build_sequence_epub(
client: &Client,
sequence: Sequence,
sequence_id: &str,
cache_options: &CacheOptions,
progress: ProgressBar,
) -> Result<EpubBuilder<ZipLibrary>, Box<dyn Error>> {
let mut builder = EpubBuilder::new(ZipLibrary::new()?)?;
builder.epub_version(epub_builder::EpubVersion::V30);
let mut level = 0;
let author = if let Some(user) = sequence.user {
if let Some(display_name) = user.displayName {
builder.metadata("author", &display_name)?;
display_name
} else if let Some(username) = user.username {
builder.metadata("author", &username)?;
username
} else {
"unknown".to_string()
}
} else {
"unknown".to_string()
};
if let Some(title) = sequence.title.clone() {
builder.metadata("title", title)?;
}
if let Some(created_at) = sequence.createdAt {
builder.set_publication_date(created_at);
}
if let Some(updated_at) = sequence.lastUpdated {
builder.set_modified_date(updated_at);
}
builder.set_uuid(Uuid::from_u128(
rapidhash::RapidRng::seed_from_u64(rapidhash::rapidhash(sequence_id.as_bytes())).random(),
));
progress.set_message(format!("{sequence_id}: Generating cover image"));
let image_id = sequence.bannerImageId.unwrap_or(
sequence
.gridImageId
.unwrap_or(DEFAULT_SEQUENCE_IMG.to_string()),
);
let authors = &[author];
let cover_image = match cover_image(
&sequence.title.clone().unwrap_or("".to_string()),
authors,
client,
&format!("{SEQUENCE_IMG_PREFIX}{image_id}"),
cache_options,
true,
)
.await
{
Ok(img) => img,
Err(e) => {
log::error!(
"error generating cover image for sequence {sequence_id}, using fallback: {e:?}"
);
cover_image(
&sequence.title.clone().unwrap_or("".to_string()),
authors,
client,
&format!("{SEQUENCE_IMG_PREFIX}{DEFAULT_SEQUENCE_IMG}"),
cache_options,
false,
)
.await?
}
};
builder.add_cover_image(COVER_PATH, cover_image.bytes.reader(), cover_image.mime)?;
let description = sequence
.contents
.and_then(|contents| contents.html)
.and_then(|html| {
rewrite_str(
&html,
RewriteStrSettings {
element_content_handlers: vec![
element!("html", |el| {
el.remove_and_keep_content();
Ok(())
}),
element!("head", |el| {
el.remove();
Ok(())
}),
element!("body", |el| {
el.remove_and_keep_content();
Ok(())
}),
],
..RewriteStrSettings::new()
},
)
.ok()
});
builder.add_content(
EpubContent::new("cover.xhtml", cover_page(COVER_PATH).as_bytes())
.reftype(ReferenceType::Cover),
)?;
let url = format!("{SEQUENCE_URL_PREFIX}{sequence_id}");
builder.add_content(
EpubContent::new(
"title.xhtml",
sequence_title_page(
&sequence
.title
.clone()
.unwrap_or(format!("untitled_{sequence_id}")),
&url,
description,
)
.as_bytes(),
)
.reftype(ReferenceType::TitlePage),
)?;
if let Some(chapters) = sequence.chapters {
for (index, chapter_option) in chapters.into_iter().enumerate() {
progress.set_message(format!("{sequence_id}: Parsing chapter {index}"));
if let Some(chapter) = chapter_option {
let chapter_id = chapter._id.unwrap();
progress.set_message(format!(
"{sequence_id}: Parsing chapter {index}: {chapter_id}"
));
if let Some(title) = chapter.title.clone() {
progress.set_message(format!(
"{sequence_id}: Parsing chapter {index} ({chapter_id}): {title}"
));
builder.add_content(
EpubContent::new(
format!("{chapter_id}_title.xhtml"),
title_page(&title).as_bytes(),
)
.title(title)
.reftype(ReferenceType::TitlePage)
.level(level),
)?;
level += 1;
}
if let Some(posts) = chapter.posts {
for (post_index, post_option) in posts.into_iter().enumerate() {
progress.set_message(format!(
"{sequence_id}: Parsing chapter {index} post {post_index}"
));
if let Some(post) = post_option {
let post_id = post._id.clone().unwrap_or_default();
progress.set_message(format!(
"{sequence_id}: Parsing chapter {index} post {post_index}: {post_id}"
));
if let Some(title) = &post.title {
progress.set_message(format!(
"{sequence_id}: Parsing chapter {index} post {post_index} ({post_id}): {title}"
));
}
if let Err(e) = async {
let mut post = post;
let id = post._id.ok_or(anyhow!("No post ID found"))?;
if let Some(fm_crosspost) = post.fmCrosspost {
if fm_crosspost.isCrosspost
&& fm_crosspost
.hostedHere
.is_some_and(|hosted_here| !hosted_here)
{
if let Some(foreign_post_id) = fm_crosspost.foreignPostId {
progress.set_message(format!(
"Fetching remote post {foreign_post_id}"
));
post = get_crosspost(
client,
&foreign_post_id,
cache_options,
)
.await?;
}
}
}
let post_title = post.title.unwrap_or(id.clone());
let mut html_body =
post.htmlBody.ok_or(anyhow!("No html body found"))?;
let mut urls = HashSet::default();
edit_image_urls(&html_body, |url| {
urls.insert(url.clone());
url
});
let mut images: HashMap<String, String> = HashMap::default();
for url in urls {
match intern_image(&mut builder, client, &url, cache_options)
.await
{
Ok(new_path) => {
images.insert(url, new_path);
}
Err(e) => {
log::warn!(
"Notice: Failed to intern image for post {id}: {e}"
);
images.insert(url.clone(), url);
}
}
}
html_body = edit_image_urls(&html_body, |url| images[&url].clone());
let url = format!("{LW_POST_URL_PREFIX}{id}");
builder.add_content(
EpubContent::new(
format!("{id}_content.xhtml"),
wrap_body(html_body, &post_title, &url).as_bytes(),
)
.title(post_title)
.reftype(ReferenceType::Text)
.level(level),
)?;
Ok::<(), Box<dyn Error>>(())
}
.await
{
log::error!(
"Error appending post {post_id} to sequence {sequence_id}: {e:?}"
);
}
}
}
}
if chapter.title.is_some() {
level -= 1;
}
}
}
}
Ok(builder)
}
pub fn save_sequence_epub(
builder: EpubBuilder<ZipLibrary>,
sequence: Sequence,
sequence_id: &str,
output_dir: &Path,
collection_subfolder: bool,
) -> Result<(), Box<dyn Error>> {
let mut file: Vec<u8> = vec![];
builder.generate(&mut file)?;
let title = sequence.title.unwrap_or(format!("untitled_{sequence_id}"));
let slug = slugify(title);
let out_dir = if collection_subfolder {
let folder = slugify(
sequence
.canonicalCollection
.unwrap_or(Collection {
_id: Some(String::new()),
slug: Some("library".to_string()),
title: Some("Library".to_string()),
gridImageId: None,
})
.slug
.unwrap_or("unknown".to_string()),
);
output_dir.join(folder)
} else {
output_dir.to_path_buf()
};
fs::create_dir_all(&out_dir)?;
fs::write(out_dir.join(format!("{slug}.epub")), file)?;
Ok(())
}
pub async fn build_and_save_sequence_epub(
client: &Client,
sequence: Sequence,
sequence_id: &str,
cache_options: &CacheOptions,
output_dir: &Path,
collection_subfolder: bool,
progress: ProgressBar,
) -> Result<(), Box<dyn Error>> {
let title = sequence
.clone()
.title
.unwrap_or(format!("untitled_{sequence_id}"));
let out_dir = if collection_subfolder {
let folder = slugify(
sequence
.clone()
.canonicalCollection
.unwrap_or(Collection {
_id: Some(String::new()),
slug: Some("library".to_string()),
title: Some("Library".to_string()),
gridImageId: None,
})
.slug
.unwrap_or("unknown".to_string()),
);
output_dir.join(folder)
} else {
output_dir.to_path_buf()
};
match parse_saved_sequence_epub_data(get_saved_epub_data(&title, &out_dir), &sequence) {
EpubStatus::Nonexistent => {
log::debug!("Downloading new sequence {sequence_id}: {title}");
let builder = build_sequence_epub(
client,
sequence.clone(),
sequence_id,
cache_options,
progress,
)
.await?;
save_sequence_epub(
builder,
sequence,
sequence_id,
output_dir,
collection_subfolder,
)?;
}
EpubStatus::OutOfDate => {
log::debug!("Ignoring cache for {sequence_id}: Generated copy is out-of-date.");
let builder = build_sequence_epub(
client,
sequence.clone(),
sequence_id,
&CacheOptions {
ignore_cache: true,
cache_dir: cache_options.cache_dir.clone(),
},
progress,
)
.await?;
save_sequence_epub(
builder,
sequence,
sequence_id,
output_dir,
collection_subfolder,
)?;
}
EpubStatus::UpToDate => {
log::debug!(
"Skipping epub generation for {sequence_id}: Generated copy is up-to-date."
);
}
};
Ok(())
}
pub async fn build_post_epub(
client: &Client,
post: Post,
post_id: &str,
cache_options: &CacheOptions,
progress: ProgressBar,
) -> Result<EpubBuilder<ZipLibrary>, Box<dyn Error>> {
let mut builder = EpubBuilder::new(ZipLibrary::new()?)?;
builder.epub_version(epub_builder::EpubVersion::V30);
let author = if let Some(user) = post.user {
if let Some(display_name) = user.displayName {
builder.metadata("author", &display_name)?;
display_name
} else if let Some(username) = user.username {
builder.metadata("author", &username)?;
username
} else {
"unknown".to_string()
}
} else {
"unknown".to_string()
};
let post_title = post.title.clone().unwrap_or(String::from("unknown"));
builder.metadata("title", &post_title)?;
if let Some(created_at) = post.createdAt {
builder.set_publication_date(created_at);
}
if let Some(updated_at) = post.modifiedAt {
builder.set_modified_date(updated_at);
}
builder.set_uuid(Uuid::from_u128(
rapidhash::RapidRng::seed_from_u64(rapidhash::rapidhash(post_id.as_bytes())).random(),
));
progress.set_message(format!("{post_id}: Generating cover image"));
let mut image_url = post.socialPreviewImageAutoUrl.unwrap_or(
post.sequence
.map(|sequence| {
sequence
.bannerImageId
.map(|banner_image_id| format!("{SEQUENCE_IMG_PREFIX}{banner_image_id}"))
.unwrap_or(
sequence
.gridImageId
.map(|grid_image_id| format!("{SEQUENCE_IMG_PREFIX}{grid_image_id}"))
.unwrap_or(DEFAULT_POST_IMG_URL.to_string()),
)
})
.unwrap_or(DEFAULT_POST_IMG_URL.to_string()),
);
if image_url == String::new() {
image_url = DEFAULT_POST_IMG_URL.to_string();
}
let authors = &[author];
let cover_image = match cover_image(
&post.title.clone().unwrap_or("".to_string()),
authors,
client,
&image_url,
cache_options,
true,
)
.await
{
Ok(img) => img,
Err(e) => {
log::error!("error generating cover image for post {post_id}, using fallback: {e:?}");
cover_image(
&post.title.clone().unwrap_or("".to_string()),
authors,
client,
DEFAULT_POST_IMG_URL,
cache_options,
false,
)
.await?
}
};
builder.add_cover_image(COVER_PATH, cover_image.bytes.reader(), cover_image.mime)?;
builder.add_content(
EpubContent::new("cover.xhtml", cover_page(COVER_PATH).as_bytes())
.reftype(ReferenceType::Cover),
)?;
let mut html_body = post.htmlBody.ok_or(anyhow!("No html body found"))?;
let mut urls = HashSet::default();
edit_image_urls(&html_body, |url| {
urls.insert(url.clone());
url
});
let mut images: HashMap<String, String> = HashMap::default();
for url in urls {
match intern_image(&mut builder, client, &url, cache_options).await {
Ok(new_path) => {
images.insert(url, new_path);
}
Err(e) => {
log::warn!("Notice: Failed to intern image for post {post_id}: {e}");
images.insert(url.clone(), url);
}
}
}
html_body = edit_image_urls(&html_body, |url| images[&url].clone());
let url = format!("{LW_POST_URL_PREFIX}{post_id}");
builder.add_content(
EpubContent::new(
"content.xhtml",
wrap_body(html_body, &post_title, &url).as_bytes(),
)
.title(post_title)
.reftype(ReferenceType::Text),
)?;
Ok(builder)
}
pub fn save_post_epub(
builder: EpubBuilder<ZipLibrary>,
post: Post,
post_id: &str,
output_dir: &Path,
) -> Result<(), Box<dyn Error>> {
let mut file: Vec<u8> = vec![];
builder.generate(&mut file)?;
let title = post.title.unwrap_or(format!("untitled_{post_id}"));
let slug = slugify(title);
fs::create_dir_all(output_dir)?;
fs::write(output_dir.join(format!("{slug}.epub")), file)?;
Ok(())
}
pub async fn build_and_save_post_epub(
client: &Client,
post: Post,
post_id: &str,
cache_options: &CacheOptions,
output_dir: &Path,
progress: ProgressBar,
) -> Result<(), Box<dyn Error>> {
let title = post.title.clone().unwrap_or(format!("untitled_{post_id}"));
match parse_saved_post_epub_data(get_saved_epub_data(&title, output_dir), &post) {
EpubStatus::Nonexistent => {
log::debug!("Downloading new post {post_id}: {title}");
let builder =
build_post_epub(client, post.clone(), post_id, cache_options, progress).await?;
save_post_epub(builder, post, post_id, output_dir)?;
}
EpubStatus::OutOfDate => {
log::debug!("Ignoring cache for {post_id}: Generated copy is out-of-date.");
let builder = build_post_epub(
client,
post.clone(),
post_id,
&CacheOptions {
ignore_cache: true,
cache_dir: cache_options.cache_dir.clone(),
},
progress,
)
.await?;
save_post_epub(builder, post, post_id, output_dir)?;
}
EpubStatus::UpToDate => {
log::debug!("Skipping epub generation for {post_id}: Generated copy is up-to-date.");
}
};
Ok(())
}
pub fn edit_image_urls(content: &str, mut f: impl FnMut(String) -> String) -> String {
rewrite_str(
content,
RewriteStrSettings {
element_content_handlers: vec![element!("img", |el| {
if let Some(url) = el.get_attribute("src") {
let new_url = f(url);
el.set_attribute("src", &new_url).unwrap();
}
Ok(())
})],
..RewriteStrSettings::default()
},
)
.unwrap()
}