1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
#![deny(
rustdoc::broken_intra_doc_links,
rustdoc::private_intra_doc_links,
rustdoc::bare_urls
)]
#![warn(missing_docs, rustdoc::unescaped_backticks)]
//! [](https://docs.rs/libwebnovel-storage)
//!
//! This is an implementation of a local repository of webnovels. It downloads
//! webnovels & places them in a coherent manner on the filesystem.
//!
//! ## What this does
//!
//! Basically, it provides data structures and method easing the work of implementing a program using [`libwebnovel`](https://crates.io/crates/libwebnovel).
//!
//! ## Example
//!
//! ```rust,no_run
//! use libwebnovel_storage::{LibraryError, LocalLibrary};
//! fn main() -> Result<(), LibraryError> {
//! let library_path = ".config/my_library/config.toml";
//! let mut library = LocalLibrary::load(library_path)?;
//! // Add to watchlist & download
//! library.add("https://www.royalroad.com/fiction/21220/mother-of-learning")?;
//!
//! // update all novels
//! let errors = library.update();
//! // or, if you want to have more control over the update process
//! // (for instance, printing a progress bar):
//! for novel in library.novels_mut() {
//! let novel_title = novel.title()?.clone();
//! for (i, result) in novel.update_iter().enumerate() {
//! if result.is_err() {
//! eprintln!(
//! "Encountered an error while updating novel {}: {}",
//! novel_title,
//! result.unwrap_err()
//! );
//! }
//! println!("novel {}: updated chapter {}", novel_title, i + 1);
//! }
//! }
//!
//! Ok(())
//! }
//! ```
//!
//! # TODO
//!
//! - [ ] a local filesystem representation for a novel library
//! - [x] bulk updates
//! - [x] bulk updates with an iterator, to offer control over looping and get
//! update information while they happen
//! - [ ] add epub generation
//!
//! ## Legal
//!
//! Without explicit refutation in the header of any file in this repository,
//! all files in this repository are considered under the terms of the AGPL-3
//! license (of which a copy can be found in the LICENSE file at the root of
//! this repository) and bearing the mention "Copyright (c) 2024 paulollivier &
//! contributors".
//!
//! Basically, please do not use this code without crediting its writer(s) or
//! for a commercial project.
use std::fmt::{Debug, Formatter};
use std::fs;
use std::fs::File;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use directories::BaseDirs;
use libwebnovel::{Backend, Backends, Chapter, ChapterParseError};
use log::{info, trace, warn};
use serde::{Deserialize, Serialize};
use url::Url;
/// Represents an error that can happen during Library operations
#[derive(thiserror::Error, Debug)]
pub enum LibraryError {
/// Wraps a [`std::io::Error`]
#[error(transparent)]
IoError(#[from] std::io::Error),
/// Wraps a [`toml::de::Error`]
#[error(transparent)]
TomlDeserializationError(#[from] toml::de::Error),
/// Wraps a [`toml::ser::Error`]
#[error(transparent)]
TomlSerializationError(#[from] toml::ser::Error),
/// Wraps an [`url::ParseError`]
#[error(transparent)]
UrlParseError(#[from] url::ParseError),
/// Wraps a [`ChapterParseError`]
#[error(transparent)]
ChapterParseError(#[from] ChapterParseError),
/// Represents a Backend Error
#[error(transparent)]
BackendError(#[from] libwebnovel::backends::BackendError),
/// Needed for accepting convert error with ?.
#[error(transparent)]
Infallible(#[from] std::convert::Infallible),
/// Returned when attempting to delete a fiction
#[error("No novel with URL \"{0}\"was found.")]
NoSuchNovel(Url),
}
/// A local disk storage.
/// ```rust
/// use tempfile::tempdir;
/// use libwebnovel_storage::LocalLibrary;
///
/// // Dummy path to a config file. If the path does not exist, a default configuration will be created
/// let config_path = "path/to/toml_config";
/// # let tempdir = tempdir().unwrap();
/// # let config_path = tempdir.path().join(env!("CARGO_PKG_NAME"));
///
/// let mut library = LocalLibrary::load(config_path).unwrap();
/// // Add & download a given URL
/// library
/// .add("https://www.royalroad.com/fiction/21220/mother-of-learning")
/// .unwrap();
/// let novel_urls_list = library.list();
/// assert_eq!(novel_urls_list.len(), 1);
/// assert_eq!(
/// &novel_urls_list[0].to_string(),
/// "https://www.royalroad.com/fiction/21220/mother-of-learning"
/// );
/// ```
#[derive(Serialize, Deserialize, Debug)]
pub struct LocalLibrary {
#[serde(skip)]
config_path: PathBuf,
library_base_path: PathBuf,
novels: Vec<Novel>,
}
impl LocalLibrary {}
impl Default for LocalLibrary {
fn default() -> Self {
let dirs = BaseDirs::new().unwrap();
Self {
config_path: dirs.config_dir().join(env!("CARGO_PKG_NAME")).to_path_buf(),
library_base_path: dirs.data_dir().join(env!("CARGO_PKG_NAME")),
novels: Vec::new(),
}
}
}
impl LocalLibrary {
/// Attempts to create a new [`Self`] from a config file.
/// ```rust
/// use libwebnovel_storage::LocalLibrary;
///
/// let config_path = "path/to/toml_config";
/// let library = LocalLibrary::load(config_path).unwrap();
/// ```
pub fn load(config_path: impl Into<PathBuf>) -> Result<Self, LibraryError> {
let config_path = config_path.into();
if !config_path.exists() {
info!("Could not find a configuration file, creating one with default values.");
return Ok(Self {
config_path,
..Default::default()
});
}
let mut config_file = File::open(config_path)?;
let mut config_str = String::new();
config_file.read_to_string(&mut config_str)?;
let config: Self = toml::from_str(&config_str)?;
Ok(config)
}
/// Saves the current config to disk.
/// ```rust
/// use std::path::Path;
///
/// use libwebnovel_storage::LocalLibrary;
///
/// let config_path_str = "/tmp/libwebnovel/config.toml";
/// let config_path = Path::new(config_path_str);
/// # use tempfile::tempdir;
/// # let tempdir = tempdir().unwrap();
/// # let config_path_str = tempdir.path().join(env!("CARGO_PKG_NAME")).join("config.toml");
/// let library = LocalLibrary::load(config_path_str.clone()).unwrap();
///
/// let config_path = Path::new(&config_path_str);
/// assert!(!config_path.exists());
/// library.persist().unwrap();
/// # println!("config_path: {:?}", config_path.display());
/// # println!("library: {:?}", library);
/// assert!(config_path.exists());
/// ```
pub fn persist(&self) -> Result<(), LibraryError> {
let toml = toml::to_string(self)?;
fs::create_dir_all(self.config_path.parent().unwrap())?;
let mut file = File::create(&self.config_path)?;
file.write_all(toml.as_bytes())?;
Ok(())
}
/// Returns the base path of the library storage
pub fn base_path(&self) -> &Path {
self.library_base_path.as_path()
}
/// Adds a new webnovel to watch. Will also call [`Novel::update`].
pub fn add(&mut self, url: &str) -> Result<String, LibraryError> {
let novel = Novel::new(url.parse::<Url>()?, self.base_path())?;
let novel_title = novel.title()?;
self.novels.push(novel);
self.persist()?;
Ok(novel_title)
}
/// Returns a list of webnovels currently watched
pub fn list(&self) -> Vec<Url> {
self.novels.iter().map(|novel| novel.url.clone()).collect()
}
/// Updates all watched novels. If at least one error has been encountered
/// during update.
pub fn update(&mut self) -> Result<(), Vec<LibraryError>> {
let mut errors = Vec::new();
for novel in self.novels.iter_mut() {
match novel.update() {
Ok(()) => {}
Err(e) => {
errors.push(e);
}
}
}
if !errors.is_empty() {
return Err(errors);
}
Ok(())
}
/// Removes a webnovel frow the watchlist and deletes local content. If
/// there are duplicates in the novels list, only the first found will be
/// removed and deleted.
pub fn remove(&mut self, url: &str) -> Result<(), LibraryError> {
let url = Url::parse(url)?;
self.novels.retain(|novel| {
if novel.url == url {
let path = novel.novel_path(&self.library_base_path);
if let Err(e) = fs::remove_dir_all(&path) {
warn!("Failed to remove directory {}: {}", path.display(), e);
}
return false;
}
true
});
Ok(())
}
/// Returns a reference to the internal novels vector.
pub fn novels(&self) -> &Vec<Novel> {
&self.novels
}
/// Returns a mutable reference to the internal novels vector
pub fn novels_mut(&mut self) -> &mut Vec<Novel> {
&mut self.novels
}
}
/// Represents a novel.
/// Stored on disk at the following path: <base_library_path>/<novel.title()>/
///
/// TODO: Detect remote novel title changes
#[derive(Serialize, Deserialize, Debug)]
#[serde(try_from = "NovelConfig", into = "NovelConfig")]
pub struct Novel {
url: Url,
path: PathBuf,
backend: Backends,
chapters: Vec<Chapter>,
}
impl Clone for Novel {
fn clone(&self) -> Self {
Self {
url: self.url.clone(),
path: self.path.clone(),
backend: Backends::new(self.url.as_ref()).unwrap(),
chapters: self.chapters.clone(),
}
}
}
/// An iterator over the update operation of a chapter. Used to be able to
/// monitor progress from your code.
///
/// ```rust
/// # use std::thread::sleep;
/// use std::time::Duration;
///
/// use libwebnovel_storage::{LibraryError, Novel};
/// use tempfile::tempdir;
/// use url::Url;
/// let path = "random/path";
/// # let dir = tempdir().unwrap();
/// # let path = dir.path();
/// let mut novel = Novel::new(
/// "https://www.royalroad.com/fiction/21220/mother-of-learning"
/// .parse::<Url>()
/// .unwrap(),
/// path,
/// )
/// .unwrap();
/// for (i, chapter_result) in novel.update_iter().enumerate() {
/// sleep(Duration::from_micros(500)); // throttleing requests
/// match chapter_result {
/// Ok(_) => {
/// println!(":) Chapter update succeded!")
/// }
/// Err(e) => {
/// println!(":'( chapter update failed: {e}")
/// }
/// }
/// }
/// assert_eq!(novel.get_local_chapter_count().unwrap(), 109);
/// ```
pub struct NovelChapterUpdateIter<'a> {
novel: &'a mut Novel,
current_chapter_index: usize,
total_chapter_count: usize,
}
impl Debug for NovelChapterUpdateIter<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
"NovelChapterUpdateIter {}/{}:\n Novel={},\n base_dir={}",
self.current_chapter_index,
self.total_chapter_count,
self.novel.url,
self.novel.path.display(),
)
}
}
impl<'a, 'b> Iterator for NovelChapterUpdateIter<'a> {
type Item = Result<(), LibraryError>;
fn next(&mut self) -> Option<Self::Item> {
trace!("{:?}", self);
if self.current_chapter_index > self.total_chapter_count {
return None;
}
let r = match self.novel.backend.get_chapter(self.current_chapter_index) {
Ok(v) => match Novel::persist_chapter(&self.novel.path, &v) {
Ok(_) => {
self.novel.chapters.push(v);
Some(Ok(()))
}
Err(e) => Some(Err(e)),
},
Err(e) => Some(Err(LibraryError::from(e))),
};
self.current_chapter_index += 1;
r
}
}
impl Novel {
/// Returns a new Novel from the given URL. Functionally equivalent to
/// [`Novel::try_from(Url)`].
pub fn new(url: impl Into<Url>, library_path: &Path) -> Result<Self, LibraryError> {
let url = url.into();
let mut novel = Self {
url: url.clone(),
path: Default::default(),
backend: Backends::new(url.as_ref())?,
chapters: vec![],
};
novel.path = novel.novel_path(library_path);
Ok(novel)
}
/// Returns the url of the novel
pub fn url(&self) -> &Url {
&self.url
}
/// Returns the title of the novel
pub fn title(&self) -> Result<String, LibraryError> {
Ok(self.backend.title()?)
}
fn persist_chapter(novel_path: &Path, chapter: &Chapter) -> Result<(), LibraryError> {
if !novel_path.exists() {
fs::create_dir_all(novel_path)?;
}
let chapter_file_name = match chapter.title() {
None => {
format!("{}.html", chapter.index())
}
Some(title) => {
format!("{}-{}.html", chapter.index(), title)
}
};
let chapter_path = novel_path.join(chapter_file_name);
let mut file = File::create(chapter_path)?;
file.write_all(chapter.to_string().as_bytes())?;
Ok(())
}
/// Fetch remote chapters & saves them locally. Will attempt to fix
/// duplicates, ordering, and detect collisions.
pub fn update(&mut self) -> Result<(), LibraryError> {
self.load_local_chapters()?;
let _errors = self
.update_iter()
.filter(Result::is_err)
.map(|r| {
let err = r.unwrap_err();
warn!("{}", err);
err
})
.collect::<Vec<_>>();
// …sort them…
self.chapters.sort_by(self.backend.get_ordering_function());
// … and remove duplicates.
self.chapters.dedup();
// update the indexes accordingly
for (i, chapter) in self.chapters.iter_mut().enumerate() {
// enumerate is 0-indexed so we must add 1 :p
let chapter_index = i + 1;
if *chapter.index() != chapter_index {
warn!("There could be a conflict in chapter {}: index was expected to be {} but was {}. Setting chapter index to expectation",chapter.title().clone().unwrap_or("<title_not_found>".to_string()),chapter_index, chapter.index());
chapter.set_index(chapter_index);
}
}
// TODO: Check for remaining duplicates/index collisions
// Write all chapters to disk
for chapter in &self.chapters {
Self::persist_chapter(&self.path, chapter)?;
}
Ok(())
}
/// Returns an iterator that will fetch the chapters, returning a Result for
/// each operation. See [`NovelChapterUpdateIter::next`] for more info.
/// Will not attempt to handle duplicates & other stuff.
pub fn update_iter<'slf>(&'slf mut self) -> NovelChapterUpdateIter<'slf> {
let total_chapter_count = self.backend.get_chapter_count().unwrap();
NovelChapterUpdateIter {
novel: self,
current_chapter_index: 1,
total_chapter_count,
}
}
/// Returns the count of locally-saved chapters
pub fn get_local_chapter_count(&self) -> Result<usize, LibraryError> {
if !self.path.exists() {
return Ok(0);
}
// count the number of chapters in self.novel_dir()
Ok(fs::read_dir(&self.path)?.count())
}
/// returns the count of remote chapters
pub fn get_remote_chapter_count(&self) -> Result<usize, LibraryError> {
Ok(self.backend.get_chapter_count()?)
}
/// Loads local chapters and saves them in `self.chapters`
fn load_local_chapters(&mut self) -> Result<(), LibraryError> {
let novel_path = &self.path;
if !novel_path.exists() {
return Ok(());
}
let chapter_files = fs::read_dir(&novel_path)?;
for chapter_file in chapter_files {
let chapter_file = chapter_file?;
let chapter_path = chapter_file.path();
let mut file = File::open(&chapter_path)?;
let mut content = String::new();
file.read_to_string(&mut content)?;
let chapter = content.parse()?;
self.chapters.push(chapter);
}
Ok(())
}
fn novel_path(&self, library_base_path: &Path) -> PathBuf {
// TODO: add a check on the `unwrap` of self.backend.title()
library_base_path.join(self.backend.title().unwrap())
}
}
impl TryFrom<Url> for Novel {
type Error = LibraryError;
/// WARNING: returns an uninitialized Novel.path!
fn try_from(value: Url) -> Result<Self, Self::Error> {
let novel = Self {
url: value.clone(),
path: Default::default(),
backend: Backends::new(value.as_str())?,
chapters: vec![],
};
Ok(novel)
}
}
impl TryFrom<NovelConfig> for Novel {
type Error = LibraryError;
fn try_from(value: NovelConfig) -> Result<Self, Self::Error> {
Self::try_from(value.url)
}
}
/// A type not destined to be used directly, but rather by serde.
#[derive(Serialize, Deserialize, Clone, Debug)]
struct NovelConfig {
url: Url,
path: PathBuf,
}
impl From<Novel> for NovelConfig {
fn from(value: Novel) -> Self {
Self {
url: value.url,
path: value.path,
}
}
}