leetcode_api/leetcode/impl_lc/
get_qs.rs

1use std::sync::atomic::Ordering;
2
3use futures::{StreamExt, stream};
4use lcode_config::global::G_USER_CONFIG;
5use miette::Result;
6use tracing::{debug, error};
7
8use crate::{
9    Json,
10    dao::{InsertToDB, query::Query, save_info::FileInfo},
11    entities::index,
12    leetcode::{
13        CATEGORIES, CUR_QS_INDEX_NUM, CUR_TOPIC_QS_INDEX_NUM, IdSlug, LeetCode, TOTAL_QS_INDEX_NUM,
14        TOTAL_TOPIC_QS_INDEX_NUM,
15        graphqls::GraphqlQuery,
16        question::{
17            pb_list::PbListData,
18            qs_detail::{Question, QuestionData},
19            qs_index::Problems,
20        },
21    },
22};
23
24impl LeetCode {
25    /// get leetcode index
26    ///
27    /// # Errors
28    ///
29    /// - network error
30    /// - leetcode url change
31    /// - `DbErr`
32    /// * `force`: when true will force update
33    pub async fn sync_problem_index(&self) -> Result<()> {
34        stream::iter(CATEGORIES)
35            .for_each_concurrent(None, |category| async move {
36                let all_pb_url = G_USER_CONFIG
37                    .urls
38                    .mod_all_pb_api(category);
39
40                // try 6 times
41                let mut count = 0;
42                let pbs: Problems = loop {
43                    match self
44                        .request(&all_pb_url, None, self.headers.clone())
45                        .await
46                    {
47                        Ok(v) => break v,
48                        Err(err) => {
49                            count += 1;
50                            error!("{}, frequency: {}", err, count);
51                            if count > 5 {
52                                break Problems::default();
53                            }
54                        },
55                    }
56                };
57
58                TOTAL_QS_INDEX_NUM.fetch_add(pbs.num_total, Ordering::Relaxed);
59
60                stream::iter(pbs.stat_status_pairs)
61                    .for_each_concurrent(None, |mut problem| async move {
62                        problem
63                            .insert_to_db(category.to_owned())
64                            .await;
65                        CUR_QS_INDEX_NUM.fetch_add(1, Ordering::Relaxed);
66                    })
67                    .await;
68            })
69            .await;
70
71        TOTAL_QS_INDEX_NUM.store(0, Ordering::Relaxed);
72        CUR_QS_INDEX_NUM.store(0, Ordering::Relaxed);
73        Ok(())
74    }
75
76    /// get question titleSlug and topicTags info
77    pub async fn sync_index_topic(&self) -> Result<()> {
78        let url = &G_USER_CONFIG.urls.graphql;
79
80        let graphql = GraphqlQuery::get_count();
81        let data: PbListData = self
82            .request(url, Some(&graphql.0), self.headers.clone())
83            .await?;
84        let total = data.data.problemset_question_list.total;
85
86        stream::iter((0..total).step_by(100))
87            .for_each_concurrent(None, |skip| async move {
88                let graphql = GraphqlQuery::new(skip);
89
90                // try 4 times
91                let mut count = 0;
92                let data: PbListData = loop {
93                    match self
94                        .request(url, Some(&graphql), self.headers.clone())
95                        .await
96                    {
97                        Ok(it) => break it,
98                        Err(err) => {
99                            count += 1;
100                            error!("{}, frequency: {}", err, count);
101                            if count > 3 {
102                                break PbListData::default();
103                            }
104                        },
105                    }
106                };
107
108                TOTAL_TOPIC_QS_INDEX_NUM.fetch_add(100, Ordering::Relaxed);
109
110                let pb_list = data
111                    .data
112                    .problemset_question_list
113                    .questions;
114
115                stream::iter(pb_list)
116                    .for_each_concurrent(None, |mut new_pb| async move {
117                        new_pb.insert_to_db(0).await;
118                        CUR_TOPIC_QS_INDEX_NUM.fetch_add(1, Ordering::Relaxed);
119                    })
120                    .await;
121            })
122            .await;
123
124        TOTAL_TOPIC_QS_INDEX_NUM.store(0, Ordering::Relaxed);
125        CUR_TOPIC_QS_INDEX_NUM.store(0, Ordering::Relaxed);
126        Ok(())
127    }
128
129    async fn get_qs_detail_helper_force(&self, pb: &index::Model) -> Result<Question> {
130        let json: Json = GraphqlQuery::qs_detail(&pb.question_title_slug);
131
132        let mut qs: QuestionData = self
133            .request(
134                &G_USER_CONFIG.urls.graphql,
135                Some(&json),
136                self.headers.clone(),
137            )
138            .await?;
139
140        qs.data.question.qs_slug = Some(pb.question_title_slug.clone());
141        qs.data
142            .question
143            .insert_one(pb.question_id)
144            .await;
145
146        Ok(qs.data.question)
147    }
148
149    /// Get the details of the problem, and if it's in the cache, use it.
150    /// Write data to file.
151    ///
152    /// * `id`: id of the problem
153    /// * `force`: when true, the cache will be re-fetched
154    /// * `write`: when true, write to file
155    pub async fn get_qs_detail(
156        &self,
157        idslug: IdSlug,
158        force: bool,
159        write: bool,
160    ) -> Result<Question> {
161        if let IdSlug::Id(id) = idslug {
162            if id == 0 {
163                miette::bail!("Question Id require large 0")
164            }
165        }
166        let pb = Query::get_question_index(&idslug).await?;
167        debug!("pb: {:?}", pb);
168        let detail = if force {
169            self.get_qs_detail_helper_force(&pb)
170                .await?
171        }
172        else {
173            let temp = Query::query_detail_by_id(pb.question_id).await?;
174
175            let the_detail = temp.unwrap_or_default();
176            let detail: Question = serde_json::from_str(&the_detail.content).unwrap_or_default();
177            // deserialize failed
178            if detail.qs_slug.is_none() {
179                self.get_qs_detail_helper_force(&pb)
180                    .await?
181            }
182            else {
183                detail
184            }
185        };
186
187        if write {
188            let chf = FileInfo::build(&pb).await?;
189            chf.write_to_file(&detail).await?;
190        }
191
192        Ok(detail)
193    }
194}
leetcode_api/leetcode/impl_lc/get_qs.rs

leetcode_api/leetcode/impl_lc/
get_qs.rs
