cktool/downloader/
get_posts_from_page.rs

1use std::{
2    thread::{self},
3    time::Duration,
4};
5
6use anyhow::{Context, Result};
7use json::JsonValue;
8
9use crate::{declare, request};
10
11use super::Downloader;
12
13impl Downloader {
14    /// Fetches all post attachments from a specific page URL
15    ///
16    /// # Arguments
17    /// * `url` - The URL of the post page
18    ///
19    /// # Returns
20    /// * `Result<Vec<String>>` - Vector of file paths to download
21    pub async fn get_posts_from_page(&mut self, url: &str) -> Result<Vec<String>> {
22        let mut posts = Vec::new();
23        let mut json_parse_retry = self.retry;
24        let mut http_retry = self.retry;
25        loop {
26            let client = request::new()?;
27            let res = match client.get(url).send().await {
28                Ok(v) => v,
29                Err(_) => {
30                    if !http_retry == 0 {
31                        http_retry -= 1;
32                        thread::sleep(Duration::from_secs(declare::ERROR_REQUEST_DELAY_SEC));
33                        continue;
34                    }
35                    return Err(anyhow::anyhow!(
36                        "Failed http request in `get_posts_from_page`"
37                    ));
38                }
39            };
40
41            let text = res
42                .text()
43                .await
44                .context("Cannot convert response body to text [res.text()]")?;
45            let obj = match json::parse(&text).context("Cannot parse JSON from response body") {
46                Ok(v) => v,
47                Err(_) => {
48                    if json_parse_retry > 0 {
49                        json_parse_retry -= 1;
50                        thread::sleep(Duration::from_secs(declare::TOO_MANY_REQUESTS_DELAY_SEC));
51                        continue;
52                    } else {
53                        break;
54                    }
55                }
56            };
57
58            // extract creator name for logs
59            if self.creator_name.lock().await.is_none()
60                && let Some(creator_name) = obj["user"]["name"].as_str()
61            {
62                *self.creator_name.lock().await = Some(creator_name.to_string());
63            }
64            let mut is_skip = false;
65            // Add attachments
66            if let JsonValue::Array(attachments) = &obj["attachments"] {
67                for atta in attachments {
68                    if atta["server"].is_null() || atta["path"].is_null() {
69                        is_skip = true;
70                    } else {
71                        posts.push(format!("{}/data{}", atta["server"], atta["path"]));
72                    }
73                }
74            }
75            // Add previews
76            if let JsonValue::Array(previews) = &obj["previews"] {
77                for preview in previews {
78                    // Some of videos could not be download, so it will be skipped.
79                    if preview["server"].is_null() || preview["path"].is_null() {
80                        if is_skip {
81                            self.info.lock().await.add_skip_file(url.to_string());
82                        }
83                    } else {
84                        posts.push(format!("{}/data{}", preview["server"], preview["path"]));
85                    }
86                }
87            }
88            break;
89        }
90        Ok(posts)
91    }
92}
cktool/downloader/get_posts_from_page.rs

cktool/downloader/
get_posts_from_page.rs
