scraper-trail 0.2.0

Scraping framework and tools
Documentation
use std::{
    marker::PhantomData,
    path::{Path, PathBuf},
};

use crate::archive::Archiveable;

#[derive(Debug, thiserror::Error)]
pub enum Error {
    #[error("I/O error")]
    Io(#[from] std::io::Error),
    #[error("JSON error")]
    Json(#[from] serde_json::Error),
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Store {
    pub base: PathBuf,
}

impl Store {
    pub fn new<P: AsRef<Path>>(base: P) -> Self {
        Self {
            base: base.as_ref().to_path_buf(),
        }
    }

    pub fn paths(&self, reverse: bool) -> Result<Vec<PathBuf>, std::io::Error> {
        let mut paths = std::fs::read_dir(&self.base)?
            .map(|entry| entry.map(|entry| entry.path()))
            .collect::<Result<Vec<_>, _>>()?;

        paths.sort();

        if reverse {
            paths.reverse();
        }

        Ok(paths)
    }

    pub fn contents(&self, reverse: bool) -> Result<Contents, std::io::Error> {
        Ok(Contents {
            // We put the paths in reverse order, since we'll be popping them off the `Vec`.
            paths: self.paths(!reverse)?,
        })
    }

    pub fn entries<T>(&self, reverse: bool) -> Result<Entries<T>, std::io::Error> {
        Ok(Entries {
            contents: self.contents(reverse)?,
            _target: PhantomData,
        })
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Contents {
    paths: Vec<PathBuf>,
}

impl Iterator for Contents {
    type Item = (PathBuf, Result<String, std::io::Error>);

    fn next(&mut self) -> Option<Self::Item> {
        self.paths.pop().map(|path| {
            let contents = std::fs::read_to_string(&path);

            (path, contents)
        })
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Entries<T> {
    contents: Contents,
    _target: PhantomData<T>,
}

use bounded_static::IntoBoundedStatic;

impl<T: Archiveable + IntoBoundedStatic> Iterator for Entries<T>
where
    T::Static: Archiveable,
    T::RequestParams: Into<<T::Static as Archiveable>::RequestParams>,
{
    type Item = (
        PathBuf,
        Result<crate::archive::entry::Entry<'static, T::Static>, Error>,
    );

    fn next(&mut self) -> Option<Self::Item> {
        self.contents.next().map(|(path, contents)| {
            let entry = contents.map_err(Error::from).and_then(|contents| {
                serde_json::from_str::<crate::archive::entry::Entry<'_, T>>(&contents)
                    .map(bounded_static::IntoBoundedStatic::into_static)
                    .map_err(Error::from)
            });

            (path, entry)
        })
    }
}