article_scraper 2.3.1

Scrap article contents from the web. Powered by fivefilters full text feed configurations & mozilla readability.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

#[cfg(feature = "embed-configs")]
use rust_embed::RustEmbed;
#[cfg(feature = "embed-configs")]
use std::borrow::Borrow;
use std::{collections::HashMap, path::Path};

use super::ConfigEntry;
use crate::util::Util;

#[cfg(feature = "embed-configs")]
#[derive(RustEmbed)]
#[folder = "ftr-site-config"]
struct EmbededConfigFiles;

pub struct ConfigCollection {
    embedded_entries: HashMap<String, ConfigEntry>,
    user_entries: HashMap<String, ConfigEntry>,
}

impl ConfigCollection {
    pub async fn parse(directory: Option<&Path>) -> Self {
        let mut user_entries = HashMap::new();
        #[cfg(feature = "embed-configs")]
        let mut embedded_entries = HashMap::new();
        #[cfg(not(feature = "embed-configs"))]
        let embedded_entries = HashMap::new();

        #[cfg(feature = "embed-configs")]
        for (file_name, entry) in EmbededConfigFiles::iter()
            .filter_map(|file_name| EmbededConfigFiles::get(&file_name).map(|e| (file_name, e)))
        {
            let entry = match ConfigEntry::parse_data(entry.data).await {
                Ok(entry) => entry,
                Err(error) => {
                    tracing::error!(%error, "Parse config data");
                    continue;
                }
            };
            let file_name: &str = file_name.borrow();
            embedded_entries.insert(file_name.to_owned(), entry);
        }

        if let Some(directory) = directory {
            // create data dir if it doesn't already exist
            if let Err(error) = std::fs::DirBuilder::new().recursive(true).create(directory) {
                tracing::warn!(
                    ?directory,
                    %error,
                    "Failed to create user config directory",
                );
            }

            if let Ok(mut dir) = tokio::fs::read_dir(directory).await {
                while let Ok(Some(entry)) = dir.next_entry().await {
                    if Util::check_extension(&entry, "txt")
                        && let Ok(config) = ConfigEntry::parse_path(&entry.path()).await
                    {
                        let file_name = entry.file_name().to_string_lossy().into_owned();
                        user_entries.insert(file_name, config);
                    }
                }
            }
        }

        Self {
            embedded_entries,
            user_entries,
        }
    }

    pub fn get(&self, key: &str) -> Option<&ConfigEntry> {
        if let Some(user_entry) = self.user_entries.get(key) {
            Some(user_entry)
        } else {
            self.embedded_entries.get(key)
        }
    }
}

#[cfg(test)]
mod tests {
    use super::ConfigCollection;
    use std::path::Path;

    #[tokio::test]
    async fn read_dir() {
        let path = Path::new("~/.local/share/news-flash/ftr-site-config");
        let _collection = ConfigCollection::parse(Some(path)).await;
    }
}