cktool 1.4.0

coomer.st and kemono.cr downloader tool
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use std::{
    thread::{self},
    time::Duration,
};

use anyhow::{Context, Result};
use json::JsonValue;

use crate::{declare, request};

use super::Downloader;

impl Downloader {
    /// Fetches all post attachments from a specific page URL
    ///
    /// # Arguments
    /// * `url` - The URL of the post page
    ///
    /// # Returns
    /// * `Result<Vec<String>>` - Vector of file paths to download
    pub async fn get_posts_from_page(&mut self, url: &str) -> Result<Vec<String>> {
        let mut posts = Vec::new();
        let mut json_parse_retry = self.retry;
        let mut http_retry = self.retry;
        loop {
            let client = request::new()?;
            let res = match client.get(url).send().await {
                Ok(v) => v,
                Err(_) => {
                    if !http_retry == 0 {
                        http_retry -= 1;
                        thread::sleep(Duration::from_secs(declare::ERROR_REQUEST_DELAY_SEC));
                        continue;
                    }
                    return Err(anyhow::anyhow!(
                        "Failed http request in `get_posts_from_page`"
                    ));
                }
            };

            let text = res
                .text()
                .await
                .context("Cannot convert response body to text [res.text()]")?;
            let obj = match json::parse(&text).context("Cannot parse JSON from response body") {
                Ok(v) => v,
                Err(_) => {
                    if json_parse_retry > 0 {
                        json_parse_retry -= 1;
                        thread::sleep(Duration::from_secs(declare::TOO_MANY_REQUESTS_DELAY_SEC));
                        continue;
                    } else {
                        break;
                    }
                }
            };

            // extract creator name for logs
            if self.creator_name.lock().await.is_none()
                && let Some(creator_name) = obj["user"]["name"].as_str()
            {
                *self.creator_name.lock().await = Some(creator_name.to_string());
            }
            let mut is_skip = false;
            // Add attachments
            if let JsonValue::Array(attachments) = &obj["attachments"] {
                for atta in attachments {
                    if atta["server"].is_null() || atta["path"].is_null() {
                        is_skip = true;
                    } else {
                        posts.push(format!("{}/data{}", atta["server"], atta["path"]));
                    }
                }
            }
            // Add previews
            if let JsonValue::Array(previews) = &obj["previews"] {
                for preview in previews {
                    // Some of videos could not be download, so it will be skipped.
                    if preview["server"].is_null() || preview["path"].is_null() {
                        if is_skip {
                            self.info.lock().await.add_skip_file(url.to_string());
                        }
                    } else {
                        posts.push(format!("{}/data{}", preview["server"], preview["path"]));
                    }
                }
            }
            break;
        }
        Ok(posts)
    }
}