kyanite 0.5.1

A modular, expandable, gallery collector.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

use crate::collector::KyaniteCollector;
use crate::item::KyaniteItem;
use log::{debug, error, info};
use serde::{Deserialize, Serialize};

#[derive(Clone, Debug, Default)]
pub struct DanbooruCollector;

impl DanbooruCollector {
    pub fn new() -> Self {
        Self::default()
    }
    pub fn boxed() -> Box<dyn KyaniteCollector> {
        Box::new(Self::new())
    }
}

#[async_trait::async_trait]
impl KyaniteCollector for DanbooruCollector {
    fn id(&self) -> &'static str {
        "danbooru"
    }

    fn name(&self) -> &'static str {
        "danbooru"
    }

    fn api_base(&self) -> &'static str {
        "https://danbooru.donmai.us/posts.json?limit=1000"
    }

    fn site_base(&self) -> &'static str {
        "https://danbooru.donmai.us"
    }

    fn tags_argument(&self) -> &'static str {
        "tags"
    }

    fn page_argument(&self) -> &'static str {
        "page"
    }

    async fn collect(&self, tags: Vec<String>) -> anyhow::Result<Vec<KyaniteItem>> {
        info!("Starting {} collector...", &self.name());
        let mut items = Vec::new();
        if tags.len() <= 2 {
            let mut page = 1u64; // Starts at 1.
            let mut finished = false;
            while !finished {
                debug!("Grabbing page with Reqwest GET...");
                let joined_tags = tags.clone().join("+");
                let resp = reqwest::get(&self.api_by_page(joined_tags, page)).await?;
                debug!("Reading the page body as text...");
                let body = resp.text().await?;
                debug!("Deserializing posts...");
                let posts: Vec<DanbooruPost> = match serde_json::from_str(&body) {
                    Ok(posts) => posts,
                    Err(why) => {
                        debug!(
                            "Failed getting page {} of {}, gracefully ending collection: {}",
                            page,
                            self.name(),
                            why
                        );
                        Vec::new()
                    }
                };
                info!(
                    "Found {} {} on page {} of {}...",
                    posts.len(),
                    if posts.len() == 1 { "post" } else { "posts" },
                    page,
                    self.name()
                );
                if posts.is_empty() {
                    finished = true;
                    info!("Page {} is empty, stopping collection.", &page);
                } else {
                    for post in posts {
                        if post.valid() {
                            items.push(KyaniteItem::new(
                                post.file_url,
                                tags.clone(),
                                self.id().to_owned(),
                            ));
                        }
                    }
                    page += 1;
                }
            }
        } else {
            error!("Danbooru disallows more than 2 tags, so it's being skipped.")
        }
        Ok(items)
    }
}

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct DanbooruPost {
    #[serde(default = "String::default")]
    pub file_url: String,
    #[serde(default = "String::default")]
    pub tag_string_general: String,
    #[serde(default = "String::default")]
    pub md5: String,
}

impl DanbooruPost {
    pub fn valid(&self) -> bool {
        !(self.file_url.is_empty() || self.md5.is_empty())
    }
}